admin/rocky-man
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

CUSP-1345 CUSP-1343 CUSP-1344 - Add older version ability

Browse Source 
Signed-off-by: Stephen Simpson <ssimpson89@users.noreply.github.com>
This commit is contained in:
Stephen Simpson
2025-12-04 15:49:42 -06:00
parent 890d7fc8f9
commit 68b9310862
4 changed files with 154 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
src/rocky_man/main.py
View File

@@ -2,6 +2,7 @@
import argparse
import logging
import re
import sys
from pathlib import Path
@@ -49,16 +50,13 @@ def process_version(config: Config, version: str, template_dir: Path) -> bool:
# Use first available architecture (man pages are arch-independent)
arch = config.architectures[0]
# Get repository URL
repo_url = config.get_repo_url(version, repo_type, arch)
# Create cache dir for this repo
cache_dir = config.download_dir / f".cache/{version}/{repo_type}"
try:
# Initialize repository manager
repo_manager = RepoManager(
repo_url=repo_url,
config=config,
version=version,
repo_type=repo_type,
arch=arch,
@@ -250,6 +248,12 @@ def main():
help="Rocky Linux mirror URL (default: http://dl.rockylinux.org/)",
)
parser.add_argument(
"--vault",
action="store_true",
help="Use vault directory instead of pub (vault/rocky instead of pub/rocky)",
)
parser.add_argument(
"--template-dir",
type=Path,
@@ -307,9 +311,13 @@ def main():
elif args.skip_languages is not None:
skip_languages = args.skip_languages
# Determine content directory
content_dir = "vault/rocky" if args.vault else "pub/rocky"
# Create configuration
config = Config(
base_url=args.mirror,
content_dir=content_dir,
versions=args.versions,
repo_types=args.repo_types,
download_dir=args.download_dir,
@@ -325,8 +333,17 @@ def main():
allow_all_sections=args.allow_all_sections,
)
# Scan for existing versions in output directory
existing_versions = [
d.name
for d in config.output_dir.iterdir()
if d.is_dir() and re.match(r"\d+\.\d+", d.name)
]
all_versions = sorted(set(existing_versions + config.versions))
logger.info("Rocky Man - Rocky Linux Man Page Generator")
logger.info(f"Versions: {', '.join(config.versions)}")
logger.info(f"Versions to process: {', '.join(config.versions)}")
logger.info(f"All known versions: {', '.join(all_versions)}")
logger.info(f"Repositories: {', '.join(config.repo_types)}")
logger.info(f"Output directory: {config.output_dir}")
@@ -360,7 +377,7 @@ def main():
# Generate root index
logger.info("Generating root index page...")
web_gen = WebGenerator(args.template_dir, config.output_dir)
web_gen.generate_root_index(processed_versions)
web_gen.generate_root_index(all_versions)
logger.info("=" * 60)
logger.info("Processing complete!")

56
src/rocky_man/repo/manager.py
View File

@@ -25,7 +25,7 @@ class RepoManager:
def __init__(
self,
repo_url: str,
config,
version: str,
repo_type: str,
arch: str,
@@ -35,14 +35,14 @@ class RepoManager:
"""Initialize repository manager.
Args:
repo_url: Full repository URL
config: Configuration object
version: Rocky Linux version (e.g., '9.5')
repo_type: Repository type ('BaseOS' or 'AppStream')
arch: Architecture (e.g., 'x86_64')
cache_dir: Directory for caching metadata
download_dir: Directory for downloading packages
"""
self.repo_url = repo_url
self.config = config
self.version = version
self.repo_type = repo_type
self.arch = arch
@@ -58,7 +58,7 @@ class RepoManager:
self.base.conf.errorlevel = 0
self.base.conf.cachedir = str(self.cache_dir / "dnf")
self._configure_repo()
self.repo_url = None
self.packages_with_manpages: Optional[Set[str]] = None
def _configure_repo(self):
@@ -88,8 +88,32 @@ class RepoManager:
if self.packages_with_manpages is not None:
return self.packages_with_manpages
parser = ContentsParser(self.repo_url, self.cache_dir)
self.packages_with_manpages = parser.get_packages_with_manpages()
# Try pub first, then vault if it fails
content_dirs = ["pub/rocky", "vault/rocky"]
for content_dir in content_dirs:
original_content_dir = self.config.content_dir
self.config.content_dir = content_dir
try:
repo_url = self.config.get_repo_url(
self.version, self.repo_type, self.arch
)
parser = ContentsParser(repo_url, self.cache_dir)
packages = parser.get_packages_with_manpages()
if packages: # Only use if it has man pages
self.packages_with_manpages = packages
self.repo_url = repo_url # Set for later use
logger.info(f"Using repository: {repo_url}")
break
else:
logger.warning(f"No man pages found in {content_dir}, trying next")
except Exception as e:
logger.warning(f"Failed to load metadata from {content_dir}: {e}")
finally:
self.config.content_dir = original_content_dir
else:
raise RuntimeError(
f"Failed to load repository metadata for {self.version} {self.repo_type} from both pub and vault"
)
return self.packages_with_manpages
@@ -102,7 +126,9 @@ class RepoManager:
Returns:
List of Package objects
"""
logger.info(f"Querying packages from {self.repo_type} ({self.version}/{self.arch})")
logger.info(
f"Querying packages from {self.repo_type} ({self.version}/{self.arch})"
)
# Get packages with man pages if filtering
manpage_packages = None
@@ -110,6 +136,9 @@ class RepoManager:
manpage_packages = self.discover_packages_with_manpages()
logger.info(f"Filtering to {len(manpage_packages)} packages with man pages")
# Configure DNF repo now that we have the correct repo_url
self._configure_repo()
packages = []
# Query all available packages
@@ -176,7 +205,7 @@ class RepoManager:
response.raise_for_status()
# Download with progress (optional: could add progress bar here)
with open(download_path, 'wb') as f:
with open(download_path, "wb") as f:
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
@@ -192,9 +221,7 @@ class RepoManager:
return False
def download_packages(
self,
packages: List[Package],
max_workers: int = 5
self, packages: List[Package], max_workers: int = 5
) -> List[Package]:
"""Download multiple packages in parallel.
@@ -210,8 +237,7 @@ class RepoManager:
with ThreadPoolExecutor(max_workers=max_workers) as executor:
# Submit all download tasks
future_to_pkg = {
executor.submit(self.download_package, pkg): pkg
for pkg in packages
executor.submit(self.download_package, pkg): pkg for pkg in packages
}
# Process completed downloads
@@ -223,7 +249,9 @@ class RepoManager:
except Exception as e:
logger.error(f"Error processing {pkg.name}: {e}")
logger.info(f"Successfully downloaded {len(downloaded)}/{len(packages)} packages")
logger.info(
f"Successfully downloaded {len(downloaded)}/{len(packages)} packages"
)
return downloaded
def cleanup_package(self, package: Package):

101
src/rocky_man/web/generator.py
View File

@@ -36,7 +36,7 @@ class WebGenerator:
# Setup Jinja2 environment
self.env = Environment(
loader=FileSystemLoader(str(self.template_dir)),
autoescape=select_autoescape(['html', 'xml'])
autoescape=select_autoescape(["html", "xml"]),
)
def generate_manpage_html(self, man_file: ManFile, version: str) -> bool:
@@ -54,7 +54,7 @@ class WebGenerator:
return False
try:
template = self.env.get_template('manpage.html')
template = self.env.get_template("manpage.html")
html = template.render(
title=f"{man_file.display_name} - {man_file.package_name} - Rocky Linux {version}",
@@ -62,8 +62,8 @@ class WebGenerator:
package_name=man_file.package_name,
version=version,
section=man_file.section,
language=man_file.language or 'en',
content=man_file.html_content
language=man_file.language or "en",
content=man_file.html_content,
)
# Ensure output path is set
@@ -72,7 +72,7 @@ class WebGenerator:
man_file.html_path.parent.mkdir(parents=True, exist_ok=True)
with open(man_file.html_path, 'w', encoding='utf-8') as f:
with open(man_file.html_path, "w", encoding="utf-8") as f:
f.write(html)
return True
@@ -92,19 +92,19 @@ class WebGenerator:
True if successful
"""
try:
template = self.env.get_template('index.html')
template = self.env.get_template("index.html")
html = template.render(
title=f"Rocky Linux {version} Man Pages",
version=version,
total_pages=len(search_data),
packages=sorted(search_data.keys())
packages=sorted(search_data.keys()),
)
index_path = self.output_dir / version / 'index.html'
index_path = self.output_dir / version / "index.html"
index_path.parent.mkdir(parents=True, exist_ok=True)
with open(index_path, 'w', encoding='utf-8') as f:
with open(index_path, "w", encoding="utf-8") as f:
f.write(html)
logger.info(f"Generated index for version {version}")
@@ -113,8 +113,10 @@ class WebGenerator:
except Exception as e:
logger.error(f"Error generating index for {version}: {e}")
return False
def generate_packages_index(self, version: str, search_data: Dict[str, Any]) -> bool:
def generate_packages_index(
self, version: str, search_data: Dict[str, Any]
) -> bool:
"""Generate full packages index page.
Args:
@@ -127,37 +129,36 @@ class WebGenerator:
try:
# Group packages by first letter
packages_by_letter = {}
for pkg_name, pages in search_data.items():
first_char = pkg_name[0].upper()
if not first_char.isalpha():
first_char = 'other'
first_char = "other"
if first_char not in packages_by_letter:
packages_by_letter[first_char] = []
packages_by_letter[first_char].append({
'name': pkg_name,
'count': len(pages)
})
packages_by_letter[first_char].append(
{"name": pkg_name, "count": len(pages)}
)
# Sort packages within each letter
for letter in packages_by_letter:
packages_by_letter[letter].sort(key=lambda x: x['name'])
packages_by_letter[letter].sort(key=lambda x: x["name"])
template = self.env.get_template('packages.html')
template = self.env.get_template("packages.html")
html = template.render(
title=f"All Packages - Rocky Linux {version}",
version=version,
total_packages=len(search_data),
packages_by_letter=packages_by_letter
packages_by_letter=packages_by_letter,
)
output_path = self.output_dir / version / 'packages.html'
output_path = self.output_dir / version / "packages.html"
output_path.parent.mkdir(parents=True, exist_ok=True)
with open(output_path, 'w', encoding='utf-8') as f:
with open(output_path, "w", encoding="utf-8") as f:
f.write(html)
logger.info(f"Generated packages index for version {version}")
@@ -168,9 +169,7 @@ class WebGenerator:
return False
def generate_search_index(
self,
man_files: List[ManFile],
version: str
self, man_files: List[ManFile], version: str
) -> Dict[str, Any]:
"""Generate search index from man files.
@@ -191,12 +190,12 @@ class WebGenerator:
# Create entry for this man page
entry = {
'name': man_file.name,
'section': man_file.section,
'display_name': man_file.display_name,
'language': man_file.language or 'en',
'url': man_file.uri_path,
'full_name': f"{man_file.package_name} - {man_file.display_name}"
"name": man_file.name,
"section": man_file.section,
"display_name": man_file.display_name,
"language": man_file.language or "en",
"url": man_file.uri_path,
"full_name": f"{man_file.package_name} - {man_file.display_name}",
}
# Use display name as key (handles duplicates with different sections)
@@ -222,18 +221,18 @@ class WebGenerator:
version_dir = self.output_dir / version
version_dir.mkdir(parents=True, exist_ok=True)
json_path = version_dir / 'search.json'
gz_path = version_dir / 'search.json.gz'
json_path = version_dir / "search.json"
gz_path = version_dir / "search.json.gz"
# Sort for consistency
sorted_index = {k: index[k] for k in sorted(index)}
# Save plain JSON
with open(json_path, 'w', encoding='utf-8') as f:
with open(json_path, "w", encoding="utf-8") as f:
json.dump(sorted_index, f, indent=2)
# Save gzipped JSON
with gzip.open(gz_path, 'wt', encoding='utf-8') as f:
with gzip.open(gz_path, "wt", encoding="utf-8") as f:
json.dump(sorted_index, f)
logger.info(f"Saved search index for {version} ({len(index)} packages)")
@@ -269,24 +268,30 @@ class WebGenerator:
True if successful
"""
try:
template = self.env.get_template('root.html')
template = self.env.get_template("root.html")
# Sort versions numerically (e.g., 8.10, 9.6, 10.0)
def version_key(v):
# Group versions by major version
major_to_minors = {}
for v in versions:
try:
parts = v.split('.')
return tuple(int(p) for p in parts)
except (ValueError, AttributeError):
return (0, 0)
major, minor = v.split(".")
major_to_minors.setdefault(major, []).append(minor)
except ValueError:
continue # Skip invalid versions
# Sort majors descending, minors descending within each major
major_groups = [
(major, sorted(major_to_minors[major], key=int, reverse=True))
for major in sorted(major_to_minors, key=int, reverse=True)
]
html = template.render(
title="Rocky Linux Man Pages",
versions=sorted(versions, key=version_key)
title="Rocky Linux Man Pages", major_groups=major_groups
)
index_path = self.output_dir / 'index.html'
index_path = self.output_dir / "index.html"
with open(index_path, 'w', encoding='utf-8') as f:
with open(index_path, "w", encoding="utf-8") as f:
f.write(html)
logger.info("Generated root index page")

41
templates/root.html
View File

@@ -15,9 +15,11 @@
height: auto;
}
.version-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
grid-template-columns: 1fr;
gap: 1.5rem;
margin-top: 2rem;
}
@@ -32,7 +34,7 @@
}
.version-grid {
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
grid-template-columns: 1fr;
gap: 1rem;
}
@@ -40,9 +42,21 @@
padding: 1.5rem;
}
.version-card.small {
padding: 0.75rem;
}
.version-card.small {
padding: 0.75rem;
}
.version-number {
font-size: 2rem;
}
.version-card.small .version-number {
font-size: 1.5rem;
}
}
@media (max-width: 480px) {
@@ -55,6 +69,10 @@
gap: 1rem;
}
.version-card.small {
padding: 0.5rem;
}
.intro {
font-size: 0.9rem;
}
@@ -71,6 +89,15 @@
display: block;
}
.version-card.small {
padding: 1rem;
opacity: 0.7;
}
.version-card.small .version-number {
font-size: 1.8rem;
}
.version-card:hover {
transform: translateY(-2px);
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.3);
@@ -134,15 +161,19 @@
<div class="version-section">
<h2>Select Version</h2>
{% for major, minors in major_groups %}
<div class="version-grid">
{% for version in versions %}
<a href="{{ version }}/index.html" class="version-card">
<div class="version-number">{{ version }}</div>
{% for minor in minors %}
<a href="{{ major }}.{{ minor }}/index.html" class="version-card{% if not loop.first %} small{% endif %}">
<div class="version-number">{{ major }}.{{ minor }}</div>
{% if loop.first %}
<div class="version-label">Rocky Linux</div>
<div class="version-browse">Browse man pages →</div>
{% endif %}
</a>
{% endfor %}
</div>
{% endfor %}
</div>
</div>
{% endblock %}