From 68b9310862be77a7e68bd1a74c2e67577bd8086a Mon Sep 17 00:00:00 2001 From: Stephen Simpson Date: Thu, 4 Dec 2025 15:49:42 -0600 Subject: [PATCH] CUSP-1345 CUSP-1343 CUSP-1344 - Add older version ability Signed-off-by: Stephen Simpson --- src/rocky_man/main.py | 29 ++++++++-- src/rocky_man/repo/manager.py | 56 +++++++++++++----- src/rocky_man/web/generator.py | 101 +++++++++++++++++---------------- templates/root.html | 41 +++++++++++-- 4 files changed, 154 insertions(+), 73 deletions(-) diff --git a/src/rocky_man/main.py b/src/rocky_man/main.py index b17a06b..a10b7c7 100644 --- a/src/rocky_man/main.py +++ b/src/rocky_man/main.py @@ -2,6 +2,7 @@ import argparse import logging +import re import sys from pathlib import Path @@ -49,16 +50,13 @@ def process_version(config: Config, version: str, template_dir: Path) -> bool: # Use first available architecture (man pages are arch-independent) arch = config.architectures[0] - # Get repository URL - repo_url = config.get_repo_url(version, repo_type, arch) - # Create cache dir for this repo cache_dir = config.download_dir / f".cache/{version}/{repo_type}" try: # Initialize repository manager repo_manager = RepoManager( - repo_url=repo_url, + config=config, version=version, repo_type=repo_type, arch=arch, @@ -250,6 +248,12 @@ def main(): help="Rocky Linux mirror URL (default: http://dl.rockylinux.org/)", ) + parser.add_argument( + "--vault", + action="store_true", + help="Use vault directory instead of pub (vault/rocky instead of pub/rocky)", + ) + parser.add_argument( "--template-dir", type=Path, @@ -307,9 +311,13 @@ def main(): elif args.skip_languages is not None: skip_languages = args.skip_languages + # Determine content directory + content_dir = "vault/rocky" if args.vault else "pub/rocky" + # Create configuration config = Config( base_url=args.mirror, + content_dir=content_dir, versions=args.versions, repo_types=args.repo_types, download_dir=args.download_dir, @@ -325,8 +333,17 @@ def main(): allow_all_sections=args.allow_all_sections, ) + # Scan for existing versions in output directory + existing_versions = [ + d.name + for d in config.output_dir.iterdir() + if d.is_dir() and re.match(r"\d+\.\d+", d.name) + ] + all_versions = sorted(set(existing_versions + config.versions)) + logger.info("Rocky Man - Rocky Linux Man Page Generator") - logger.info(f"Versions: {', '.join(config.versions)}") + logger.info(f"Versions to process: {', '.join(config.versions)}") + logger.info(f"All known versions: {', '.join(all_versions)}") logger.info(f"Repositories: {', '.join(config.repo_types)}") logger.info(f"Output directory: {config.output_dir}") @@ -360,7 +377,7 @@ def main(): # Generate root index logger.info("Generating root index page...") web_gen = WebGenerator(args.template_dir, config.output_dir) - web_gen.generate_root_index(processed_versions) + web_gen.generate_root_index(all_versions) logger.info("=" * 60) logger.info("Processing complete!") diff --git a/src/rocky_man/repo/manager.py b/src/rocky_man/repo/manager.py index b1aa7b5..5a8aebb 100644 --- a/src/rocky_man/repo/manager.py +++ b/src/rocky_man/repo/manager.py @@ -25,7 +25,7 @@ class RepoManager: def __init__( self, - repo_url: str, + config, version: str, repo_type: str, arch: str, @@ -35,14 +35,14 @@ class RepoManager: """Initialize repository manager. Args: - repo_url: Full repository URL + config: Configuration object version: Rocky Linux version (e.g., '9.5') repo_type: Repository type ('BaseOS' or 'AppStream') arch: Architecture (e.g., 'x86_64') cache_dir: Directory for caching metadata download_dir: Directory for downloading packages """ - self.repo_url = repo_url + self.config = config self.version = version self.repo_type = repo_type self.arch = arch @@ -58,7 +58,7 @@ class RepoManager: self.base.conf.errorlevel = 0 self.base.conf.cachedir = str(self.cache_dir / "dnf") - self._configure_repo() + self.repo_url = None self.packages_with_manpages: Optional[Set[str]] = None def _configure_repo(self): @@ -88,8 +88,32 @@ class RepoManager: if self.packages_with_manpages is not None: return self.packages_with_manpages - parser = ContentsParser(self.repo_url, self.cache_dir) - self.packages_with_manpages = parser.get_packages_with_manpages() + # Try pub first, then vault if it fails + content_dirs = ["pub/rocky", "vault/rocky"] + for content_dir in content_dirs: + original_content_dir = self.config.content_dir + self.config.content_dir = content_dir + try: + repo_url = self.config.get_repo_url( + self.version, self.repo_type, self.arch + ) + parser = ContentsParser(repo_url, self.cache_dir) + packages = parser.get_packages_with_manpages() + if packages: # Only use if it has man pages + self.packages_with_manpages = packages + self.repo_url = repo_url # Set for later use + logger.info(f"Using repository: {repo_url}") + break + else: + logger.warning(f"No man pages found in {content_dir}, trying next") + except Exception as e: + logger.warning(f"Failed to load metadata from {content_dir}: {e}") + finally: + self.config.content_dir = original_content_dir + else: + raise RuntimeError( + f"Failed to load repository metadata for {self.version} {self.repo_type} from both pub and vault" + ) return self.packages_with_manpages @@ -102,7 +126,9 @@ class RepoManager: Returns: List of Package objects """ - logger.info(f"Querying packages from {self.repo_type} ({self.version}/{self.arch})") + logger.info( + f"Querying packages from {self.repo_type} ({self.version}/{self.arch})" + ) # Get packages with man pages if filtering manpage_packages = None @@ -110,6 +136,9 @@ class RepoManager: manpage_packages = self.discover_packages_with_manpages() logger.info(f"Filtering to {len(manpage_packages)} packages with man pages") + # Configure DNF repo now that we have the correct repo_url + self._configure_repo() + packages = [] # Query all available packages @@ -176,7 +205,7 @@ class RepoManager: response.raise_for_status() # Download with progress (optional: could add progress bar here) - with open(download_path, 'wb') as f: + with open(download_path, "wb") as f: for chunk in response.iter_content(chunk_size=8192): if chunk: f.write(chunk) @@ -192,9 +221,7 @@ class RepoManager: return False def download_packages( - self, - packages: List[Package], - max_workers: int = 5 + self, packages: List[Package], max_workers: int = 5 ) -> List[Package]: """Download multiple packages in parallel. @@ -210,8 +237,7 @@ class RepoManager: with ThreadPoolExecutor(max_workers=max_workers) as executor: # Submit all download tasks future_to_pkg = { - executor.submit(self.download_package, pkg): pkg - for pkg in packages + executor.submit(self.download_package, pkg): pkg for pkg in packages } # Process completed downloads @@ -223,7 +249,9 @@ class RepoManager: except Exception as e: logger.error(f"Error processing {pkg.name}: {e}") - logger.info(f"Successfully downloaded {len(downloaded)}/{len(packages)} packages") + logger.info( + f"Successfully downloaded {len(downloaded)}/{len(packages)} packages" + ) return downloaded def cleanup_package(self, package: Package): diff --git a/src/rocky_man/web/generator.py b/src/rocky_man/web/generator.py index fba528f..23ed16e 100644 --- a/src/rocky_man/web/generator.py +++ b/src/rocky_man/web/generator.py @@ -36,7 +36,7 @@ class WebGenerator: # Setup Jinja2 environment self.env = Environment( loader=FileSystemLoader(str(self.template_dir)), - autoescape=select_autoescape(['html', 'xml']) + autoescape=select_autoescape(["html", "xml"]), ) def generate_manpage_html(self, man_file: ManFile, version: str) -> bool: @@ -54,7 +54,7 @@ class WebGenerator: return False try: - template = self.env.get_template('manpage.html') + template = self.env.get_template("manpage.html") html = template.render( title=f"{man_file.display_name} - {man_file.package_name} - Rocky Linux {version}", @@ -62,8 +62,8 @@ class WebGenerator: package_name=man_file.package_name, version=version, section=man_file.section, - language=man_file.language or 'en', - content=man_file.html_content + language=man_file.language or "en", + content=man_file.html_content, ) # Ensure output path is set @@ -72,7 +72,7 @@ class WebGenerator: man_file.html_path.parent.mkdir(parents=True, exist_ok=True) - with open(man_file.html_path, 'w', encoding='utf-8') as f: + with open(man_file.html_path, "w", encoding="utf-8") as f: f.write(html) return True @@ -92,19 +92,19 @@ class WebGenerator: True if successful """ try: - template = self.env.get_template('index.html') + template = self.env.get_template("index.html") html = template.render( title=f"Rocky Linux {version} Man Pages", version=version, total_pages=len(search_data), - packages=sorted(search_data.keys()) + packages=sorted(search_data.keys()), ) - index_path = self.output_dir / version / 'index.html' + index_path = self.output_dir / version / "index.html" index_path.parent.mkdir(parents=True, exist_ok=True) - with open(index_path, 'w', encoding='utf-8') as f: + with open(index_path, "w", encoding="utf-8") as f: f.write(html) logger.info(f"Generated index for version {version}") @@ -113,8 +113,10 @@ class WebGenerator: except Exception as e: logger.error(f"Error generating index for {version}: {e}") return False - - def generate_packages_index(self, version: str, search_data: Dict[str, Any]) -> bool: + + def generate_packages_index( + self, version: str, search_data: Dict[str, Any] + ) -> bool: """Generate full packages index page. Args: @@ -127,37 +129,36 @@ class WebGenerator: try: # Group packages by first letter packages_by_letter = {} - + for pkg_name, pages in search_data.items(): first_char = pkg_name[0].upper() if not first_char.isalpha(): - first_char = 'other' - + first_char = "other" + if first_char not in packages_by_letter: packages_by_letter[first_char] = [] - - packages_by_letter[first_char].append({ - 'name': pkg_name, - 'count': len(pages) - }) + + packages_by_letter[first_char].append( + {"name": pkg_name, "count": len(pages)} + ) # Sort packages within each letter for letter in packages_by_letter: - packages_by_letter[letter].sort(key=lambda x: x['name']) + packages_by_letter[letter].sort(key=lambda x: x["name"]) - template = self.env.get_template('packages.html') + template = self.env.get_template("packages.html") html = template.render( title=f"All Packages - Rocky Linux {version}", version=version, total_packages=len(search_data), - packages_by_letter=packages_by_letter + packages_by_letter=packages_by_letter, ) - output_path = self.output_dir / version / 'packages.html' + output_path = self.output_dir / version / "packages.html" output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, 'w', encoding='utf-8') as f: + with open(output_path, "w", encoding="utf-8") as f: f.write(html) logger.info(f"Generated packages index for version {version}") @@ -168,9 +169,7 @@ class WebGenerator: return False def generate_search_index( - self, - man_files: List[ManFile], - version: str + self, man_files: List[ManFile], version: str ) -> Dict[str, Any]: """Generate search index from man files. @@ -191,12 +190,12 @@ class WebGenerator: # Create entry for this man page entry = { - 'name': man_file.name, - 'section': man_file.section, - 'display_name': man_file.display_name, - 'language': man_file.language or 'en', - 'url': man_file.uri_path, - 'full_name': f"{man_file.package_name} - {man_file.display_name}" + "name": man_file.name, + "section": man_file.section, + "display_name": man_file.display_name, + "language": man_file.language or "en", + "url": man_file.uri_path, + "full_name": f"{man_file.package_name} - {man_file.display_name}", } # Use display name as key (handles duplicates with different sections) @@ -222,18 +221,18 @@ class WebGenerator: version_dir = self.output_dir / version version_dir.mkdir(parents=True, exist_ok=True) - json_path = version_dir / 'search.json' - gz_path = version_dir / 'search.json.gz' + json_path = version_dir / "search.json" + gz_path = version_dir / "search.json.gz" # Sort for consistency sorted_index = {k: index[k] for k in sorted(index)} # Save plain JSON - with open(json_path, 'w', encoding='utf-8') as f: + with open(json_path, "w", encoding="utf-8") as f: json.dump(sorted_index, f, indent=2) # Save gzipped JSON - with gzip.open(gz_path, 'wt', encoding='utf-8') as f: + with gzip.open(gz_path, "wt", encoding="utf-8") as f: json.dump(sorted_index, f) logger.info(f"Saved search index for {version} ({len(index)} packages)") @@ -269,24 +268,30 @@ class WebGenerator: True if successful """ try: - template = self.env.get_template('root.html') + template = self.env.get_template("root.html") - # Sort versions numerically (e.g., 8.10, 9.6, 10.0) - def version_key(v): + # Group versions by major version + major_to_minors = {} + for v in versions: try: - parts = v.split('.') - return tuple(int(p) for p in parts) - except (ValueError, AttributeError): - return (0, 0) + major, minor = v.split(".") + major_to_minors.setdefault(major, []).append(minor) + except ValueError: + continue # Skip invalid versions + + # Sort majors descending, minors descending within each major + major_groups = [ + (major, sorted(major_to_minors[major], key=int, reverse=True)) + for major in sorted(major_to_minors, key=int, reverse=True) + ] html = template.render( - title="Rocky Linux Man Pages", - versions=sorted(versions, key=version_key) + title="Rocky Linux Man Pages", major_groups=major_groups ) - index_path = self.output_dir / 'index.html' + index_path = self.output_dir / "index.html" - with open(index_path, 'w', encoding='utf-8') as f: + with open(index_path, "w", encoding="utf-8") as f: f.write(html) logger.info("Generated root index page") diff --git a/templates/root.html b/templates/root.html index e8880ee..381cdca 100644 --- a/templates/root.html +++ b/templates/root.html @@ -15,9 +15,11 @@ height: auto; } + + .version-grid { display: grid; - grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + grid-template-columns: 1fr; gap: 1.5rem; margin-top: 2rem; } @@ -32,7 +34,7 @@ } .version-grid { - grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + grid-template-columns: 1fr; gap: 1rem; } @@ -40,9 +42,21 @@ padding: 1.5rem; } + .version-card.small { + padding: 0.75rem; + } + + .version-card.small { + padding: 0.75rem; + } + .version-number { font-size: 2rem; } + + .version-card.small .version-number { + font-size: 1.5rem; + } } @media (max-width: 480px) { @@ -55,6 +69,10 @@ gap: 1rem; } + .version-card.small { + padding: 0.5rem; + } + .intro { font-size: 0.9rem; } @@ -71,6 +89,15 @@ display: block; } +.version-card.small { + padding: 1rem; + opacity: 0.7; +} + +.version-card.small .version-number { + font-size: 1.8rem; +} + .version-card:hover { transform: translateY(-2px); box-shadow: 0 4px 8px rgba(0, 0, 0, 0.3); @@ -134,15 +161,19 @@

Select Version

+ {% for major, minors in major_groups %}
- {% for version in versions %} - -
{{ version }}
+ {% for minor in minors %} + +
{{ major }}.{{ minor }}
+ {% if loop.first %}
Rocky Linux
Browse man pages →
+ {% endif %} {% endfor %}
+ {% endfor %}
{% endblock %}