add-feedback-improvements #5

Open
admin wants to merge 10 commits from add-feedback-improvements into main
4 changed files with 154 additions and 73 deletions
Showing only changes of commit 68b9310862 - Show all commits

View File

@@ -2,6 +2,7 @@
import argparse import argparse
import logging import logging
import re
import sys import sys
from pathlib import Path from pathlib import Path
@@ -49,16 +50,13 @@ def process_version(config: Config, version: str, template_dir: Path) -> bool:
# Use first available architecture (man pages are arch-independent) # Use first available architecture (man pages are arch-independent)
arch = config.architectures[0] arch = config.architectures[0]
# Get repository URL
repo_url = config.get_repo_url(version, repo_type, arch)
# Create cache dir for this repo # Create cache dir for this repo
cache_dir = config.download_dir / f".cache/{version}/{repo_type}" cache_dir = config.download_dir / f".cache/{version}/{repo_type}"
try: try:
# Initialize repository manager # Initialize repository manager
repo_manager = RepoManager( repo_manager = RepoManager(
repo_url=repo_url, config=config,
version=version, version=version,
repo_type=repo_type, repo_type=repo_type,
arch=arch, arch=arch,
@@ -250,6 +248,12 @@ def main():
help="Rocky Linux mirror URL (default: http://dl.rockylinux.org/)", help="Rocky Linux mirror URL (default: http://dl.rockylinux.org/)",
) )
parser.add_argument(
"--vault",
action="store_true",
help="Use vault directory instead of pub (vault/rocky instead of pub/rocky)",
)
parser.add_argument( parser.add_argument(
"--template-dir", "--template-dir",
type=Path, type=Path,
@@ -307,9 +311,13 @@ def main():
elif args.skip_languages is not None: elif args.skip_languages is not None:
skip_languages = args.skip_languages skip_languages = args.skip_languages
# Determine content directory
content_dir = "vault/rocky" if args.vault else "pub/rocky"
# Create configuration # Create configuration
config = Config( config = Config(
base_url=args.mirror, base_url=args.mirror,
content_dir=content_dir,
versions=args.versions, versions=args.versions,
repo_types=args.repo_types, repo_types=args.repo_types,
download_dir=args.download_dir, download_dir=args.download_dir,
@@ -325,8 +333,17 @@ def main():
allow_all_sections=args.allow_all_sections, allow_all_sections=args.allow_all_sections,
) )
# Scan for existing versions in output directory
existing_versions = [
d.name
for d in config.output_dir.iterdir()
if d.is_dir() and re.match(r"\d+\.\d+", d.name)
]
all_versions = sorted(set(existing_versions + config.versions))
logger.info("Rocky Man - Rocky Linux Man Page Generator") logger.info("Rocky Man - Rocky Linux Man Page Generator")
logger.info(f"Versions: {', '.join(config.versions)}") logger.info(f"Versions to process: {', '.join(config.versions)}")
logger.info(f"All known versions: {', '.join(all_versions)}")
logger.info(f"Repositories: {', '.join(config.repo_types)}") logger.info(f"Repositories: {', '.join(config.repo_types)}")
logger.info(f"Output directory: {config.output_dir}") logger.info(f"Output directory: {config.output_dir}")
@@ -360,7 +377,7 @@ def main():
# Generate root index # Generate root index
logger.info("Generating root index page...") logger.info("Generating root index page...")
web_gen = WebGenerator(args.template_dir, config.output_dir) web_gen = WebGenerator(args.template_dir, config.output_dir)
web_gen.generate_root_index(processed_versions) web_gen.generate_root_index(all_versions)
logger.info("=" * 60) logger.info("=" * 60)
logger.info("Processing complete!") logger.info("Processing complete!")

View File

@@ -25,7 +25,7 @@ class RepoManager:
def __init__( def __init__(
self, self,
repo_url: str, config,
version: str, version: str,
repo_type: str, repo_type: str,
arch: str, arch: str,
@@ -35,14 +35,14 @@ class RepoManager:
"""Initialize repository manager. """Initialize repository manager.
Args: Args:
repo_url: Full repository URL config: Configuration object
version: Rocky Linux version (e.g., '9.5') version: Rocky Linux version (e.g., '9.5')
repo_type: Repository type ('BaseOS' or 'AppStream') repo_type: Repository type ('BaseOS' or 'AppStream')
arch: Architecture (e.g., 'x86_64') arch: Architecture (e.g., 'x86_64')
cache_dir: Directory for caching metadata cache_dir: Directory for caching metadata
download_dir: Directory for downloading packages download_dir: Directory for downloading packages
""" """
self.repo_url = repo_url self.config = config
self.version = version self.version = version
self.repo_type = repo_type self.repo_type = repo_type
self.arch = arch self.arch = arch
@@ -58,7 +58,7 @@ class RepoManager:
self.base.conf.errorlevel = 0 self.base.conf.errorlevel = 0
self.base.conf.cachedir = str(self.cache_dir / "dnf") self.base.conf.cachedir = str(self.cache_dir / "dnf")
self._configure_repo() self.repo_url = None
self.packages_with_manpages: Optional[Set[str]] = None self.packages_with_manpages: Optional[Set[str]] = None
def _configure_repo(self): def _configure_repo(self):
@@ -88,8 +88,32 @@ class RepoManager:
if self.packages_with_manpages is not None: if self.packages_with_manpages is not None:
return self.packages_with_manpages return self.packages_with_manpages
parser = ContentsParser(self.repo_url, self.cache_dir) # Try pub first, then vault if it fails
self.packages_with_manpages = parser.get_packages_with_manpages() content_dirs = ["pub/rocky", "vault/rocky"]
for content_dir in content_dirs:
original_content_dir = self.config.content_dir
self.config.content_dir = content_dir
try:
repo_url = self.config.get_repo_url(
self.version, self.repo_type, self.arch
)
parser = ContentsParser(repo_url, self.cache_dir)
packages = parser.get_packages_with_manpages()
if packages: # Only use if it has man pages
self.packages_with_manpages = packages
self.repo_url = repo_url # Set for later use
logger.info(f"Using repository: {repo_url}")
break
else:
logger.warning(f"No man pages found in {content_dir}, trying next")
except Exception as e:
logger.warning(f"Failed to load metadata from {content_dir}: {e}")
finally:
self.config.content_dir = original_content_dir
else:
raise RuntimeError(
f"Failed to load repository metadata for {self.version} {self.repo_type} from both pub and vault"
)
return self.packages_with_manpages return self.packages_with_manpages
@@ -102,7 +126,9 @@ class RepoManager:
Returns: Returns:
List of Package objects List of Package objects
""" """
logger.info(f"Querying packages from {self.repo_type} ({self.version}/{self.arch})") logger.info(
f"Querying packages from {self.repo_type} ({self.version}/{self.arch})"
)
# Get packages with man pages if filtering # Get packages with man pages if filtering
manpage_packages = None manpage_packages = None
@@ -110,6 +136,9 @@ class RepoManager:
manpage_packages = self.discover_packages_with_manpages() manpage_packages = self.discover_packages_with_manpages()
logger.info(f"Filtering to {len(manpage_packages)} packages with man pages") logger.info(f"Filtering to {len(manpage_packages)} packages with man pages")
# Configure DNF repo now that we have the correct repo_url
self._configure_repo()
packages = [] packages = []
# Query all available packages # Query all available packages
@@ -176,7 +205,7 @@ class RepoManager:
response.raise_for_status() response.raise_for_status()
# Download with progress (optional: could add progress bar here) # Download with progress (optional: could add progress bar here)
with open(download_path, 'wb') as f: with open(download_path, "wb") as f:
for chunk in response.iter_content(chunk_size=8192): for chunk in response.iter_content(chunk_size=8192):
if chunk: if chunk:
f.write(chunk) f.write(chunk)
@@ -192,9 +221,7 @@ class RepoManager:
return False return False
def download_packages( def download_packages(
self, self, packages: List[Package], max_workers: int = 5
packages: List[Package],
max_workers: int = 5
) -> List[Package]: ) -> List[Package]:
"""Download multiple packages in parallel. """Download multiple packages in parallel.
@@ -210,8 +237,7 @@ class RepoManager:
with ThreadPoolExecutor(max_workers=max_workers) as executor: with ThreadPoolExecutor(max_workers=max_workers) as executor:
# Submit all download tasks # Submit all download tasks
future_to_pkg = { future_to_pkg = {
executor.submit(self.download_package, pkg): pkg executor.submit(self.download_package, pkg): pkg for pkg in packages
for pkg in packages
} }
# Process completed downloads # Process completed downloads
@@ -223,7 +249,9 @@ class RepoManager:
except Exception as e: except Exception as e:
logger.error(f"Error processing {pkg.name}: {e}") logger.error(f"Error processing {pkg.name}: {e}")
logger.info(f"Successfully downloaded {len(downloaded)}/{len(packages)} packages") logger.info(
f"Successfully downloaded {len(downloaded)}/{len(packages)} packages"
)
return downloaded return downloaded
def cleanup_package(self, package: Package): def cleanup_package(self, package: Package):

View File

@@ -36,7 +36,7 @@ class WebGenerator:
# Setup Jinja2 environment # Setup Jinja2 environment
self.env = Environment( self.env = Environment(
loader=FileSystemLoader(str(self.template_dir)), loader=FileSystemLoader(str(self.template_dir)),
autoescape=select_autoescape(['html', 'xml']) autoescape=select_autoescape(["html", "xml"]),
) )
def generate_manpage_html(self, man_file: ManFile, version: str) -> bool: def generate_manpage_html(self, man_file: ManFile, version: str) -> bool:
@@ -54,7 +54,7 @@ class WebGenerator:
return False return False
try: try:
template = self.env.get_template('manpage.html') template = self.env.get_template("manpage.html")
html = template.render( html = template.render(
title=f"{man_file.display_name} - {man_file.package_name} - Rocky Linux {version}", title=f"{man_file.display_name} - {man_file.package_name} - Rocky Linux {version}",
@@ -62,8 +62,8 @@ class WebGenerator:
package_name=man_file.package_name, package_name=man_file.package_name,
version=version, version=version,
section=man_file.section, section=man_file.section,
language=man_file.language or 'en', language=man_file.language or "en",
content=man_file.html_content content=man_file.html_content,
) )
# Ensure output path is set # Ensure output path is set
@@ -72,7 +72,7 @@ class WebGenerator:
man_file.html_path.parent.mkdir(parents=True, exist_ok=True) man_file.html_path.parent.mkdir(parents=True, exist_ok=True)
with open(man_file.html_path, 'w', encoding='utf-8') as f: with open(man_file.html_path, "w", encoding="utf-8") as f:
f.write(html) f.write(html)
return True return True
@@ -92,19 +92,19 @@ class WebGenerator:
True if successful True if successful
""" """
try: try:
template = self.env.get_template('index.html') template = self.env.get_template("index.html")
html = template.render( html = template.render(
title=f"Rocky Linux {version} Man Pages", title=f"Rocky Linux {version} Man Pages",
version=version, version=version,
total_pages=len(search_data), total_pages=len(search_data),
packages=sorted(search_data.keys()) packages=sorted(search_data.keys()),
) )
index_path = self.output_dir / version / 'index.html' index_path = self.output_dir / version / "index.html"
index_path.parent.mkdir(parents=True, exist_ok=True) index_path.parent.mkdir(parents=True, exist_ok=True)
with open(index_path, 'w', encoding='utf-8') as f: with open(index_path, "w", encoding="utf-8") as f:
f.write(html) f.write(html)
logger.info(f"Generated index for version {version}") logger.info(f"Generated index for version {version}")
@@ -114,7 +114,9 @@ class WebGenerator:
logger.error(f"Error generating index for {version}: {e}") logger.error(f"Error generating index for {version}: {e}")
return False return False
def generate_packages_index(self, version: str, search_data: Dict[str, Any]) -> bool: def generate_packages_index(
self, version: str, search_data: Dict[str, Any]
) -> bool:
"""Generate full packages index page. """Generate full packages index page.
Args: Args:
@@ -131,33 +133,32 @@ class WebGenerator:
for pkg_name, pages in search_data.items(): for pkg_name, pages in search_data.items():
first_char = pkg_name[0].upper() first_char = pkg_name[0].upper()
if not first_char.isalpha(): if not first_char.isalpha():
first_char = 'other' first_char = "other"
if first_char not in packages_by_letter: if first_char not in packages_by_letter:
packages_by_letter[first_char] = [] packages_by_letter[first_char] = []
packages_by_letter[first_char].append({ packages_by_letter[first_char].append(
'name': pkg_name, {"name": pkg_name, "count": len(pages)}
'count': len(pages) )
})
# Sort packages within each letter # Sort packages within each letter
for letter in packages_by_letter: for letter in packages_by_letter:
packages_by_letter[letter].sort(key=lambda x: x['name']) packages_by_letter[letter].sort(key=lambda x: x["name"])
template = self.env.get_template('packages.html') template = self.env.get_template("packages.html")
html = template.render( html = template.render(
title=f"All Packages - Rocky Linux {version}", title=f"All Packages - Rocky Linux {version}",
version=version, version=version,
total_packages=len(search_data), total_packages=len(search_data),
packages_by_letter=packages_by_letter packages_by_letter=packages_by_letter,
) )
output_path = self.output_dir / version / 'packages.html' output_path = self.output_dir / version / "packages.html"
output_path.parent.mkdir(parents=True, exist_ok=True) output_path.parent.mkdir(parents=True, exist_ok=True)
with open(output_path, 'w', encoding='utf-8') as f: with open(output_path, "w", encoding="utf-8") as f:
f.write(html) f.write(html)
logger.info(f"Generated packages index for version {version}") logger.info(f"Generated packages index for version {version}")
@@ -168,9 +169,7 @@ class WebGenerator:
return False return False
def generate_search_index( def generate_search_index(
self, self, man_files: List[ManFile], version: str
man_files: List[ManFile],
version: str
) -> Dict[str, Any]: ) -> Dict[str, Any]:
"""Generate search index from man files. """Generate search index from man files.
@@ -191,12 +190,12 @@ class WebGenerator:
# Create entry for this man page # Create entry for this man page
entry = { entry = {
'name': man_file.name, "name": man_file.name,
'section': man_file.section, "section": man_file.section,
'display_name': man_file.display_name, "display_name": man_file.display_name,
'language': man_file.language or 'en', "language": man_file.language or "en",
'url': man_file.uri_path, "url": man_file.uri_path,
'full_name': f"{man_file.package_name} - {man_file.display_name}" "full_name": f"{man_file.package_name} - {man_file.display_name}",
} }
# Use display name as key (handles duplicates with different sections) # Use display name as key (handles duplicates with different sections)
@@ -222,18 +221,18 @@ class WebGenerator:
version_dir = self.output_dir / version version_dir = self.output_dir / version
version_dir.mkdir(parents=True, exist_ok=True) version_dir.mkdir(parents=True, exist_ok=True)
json_path = version_dir / 'search.json' json_path = version_dir / "search.json"
gz_path = version_dir / 'search.json.gz' gz_path = version_dir / "search.json.gz"
# Sort for consistency # Sort for consistency
sorted_index = {k: index[k] for k in sorted(index)} sorted_index = {k: index[k] for k in sorted(index)}
# Save plain JSON # Save plain JSON
with open(json_path, 'w', encoding='utf-8') as f: with open(json_path, "w", encoding="utf-8") as f:
json.dump(sorted_index, f, indent=2) json.dump(sorted_index, f, indent=2)
# Save gzipped JSON # Save gzipped JSON
with gzip.open(gz_path, 'wt', encoding='utf-8') as f: with gzip.open(gz_path, "wt", encoding="utf-8") as f:
json.dump(sorted_index, f) json.dump(sorted_index, f)
logger.info(f"Saved search index for {version} ({len(index)} packages)") logger.info(f"Saved search index for {version} ({len(index)} packages)")
@@ -269,24 +268,30 @@ class WebGenerator:
True if successful True if successful
""" """
try: try:
template = self.env.get_template('root.html') template = self.env.get_template("root.html")
# Sort versions numerically (e.g., 8.10, 9.6, 10.0) # Group versions by major version
def version_key(v): major_to_minors = {}
for v in versions:
try: try:
parts = v.split('.') major, minor = v.split(".")
return tuple(int(p) for p in parts) major_to_minors.setdefault(major, []).append(minor)
except (ValueError, AttributeError): except ValueError:
return (0, 0) continue # Skip invalid versions
# Sort majors descending, minors descending within each major
major_groups = [
(major, sorted(major_to_minors[major], key=int, reverse=True))
for major in sorted(major_to_minors, key=int, reverse=True)
]
html = template.render( html = template.render(
title="Rocky Linux Man Pages", title="Rocky Linux Man Pages", major_groups=major_groups
versions=sorted(versions, key=version_key)
) )
index_path = self.output_dir / 'index.html' index_path = self.output_dir / "index.html"
with open(index_path, 'w', encoding='utf-8') as f: with open(index_path, "w", encoding="utf-8") as f:
f.write(html) f.write(html)
logger.info("Generated root index page") logger.info("Generated root index page")

View File

@@ -15,9 +15,11 @@
height: auto; height: auto;
} }
.version-grid { .version-grid {
display: grid; display: grid;
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); grid-template-columns: 1fr;
gap: 1.5rem; gap: 1.5rem;
margin-top: 2rem; margin-top: 2rem;
} }
@@ -32,7 +34,7 @@
} }
.version-grid { .version-grid {
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); grid-template-columns: 1fr;
gap: 1rem; gap: 1rem;
} }
@@ -40,9 +42,21 @@
padding: 1.5rem; padding: 1.5rem;
} }
.version-card.small {
padding: 0.75rem;
}
.version-card.small {
padding: 0.75rem;
}
.version-number { .version-number {
font-size: 2rem; font-size: 2rem;
} }
.version-card.small .version-number {
font-size: 1.5rem;
}
} }
@media (max-width: 480px) { @media (max-width: 480px) {
@@ -55,6 +69,10 @@
gap: 1rem; gap: 1rem;
} }
.version-card.small {
padding: 0.5rem;
}
.intro { .intro {
font-size: 0.9rem; font-size: 0.9rem;
} }
@@ -71,6 +89,15 @@
display: block; display: block;
} }
.version-card.small {
padding: 1rem;
opacity: 0.7;
}
.version-card.small .version-number {
font-size: 1.8rem;
}
.version-card:hover { .version-card:hover {
transform: translateY(-2px); transform: translateY(-2px);
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.3); box-shadow: 0 4px 8px rgba(0, 0, 0, 0.3);
@@ -134,15 +161,19 @@
<div class="version-section"> <div class="version-section">
<h2>Select Version</h2> <h2>Select Version</h2>
{% for major, minors in major_groups %}
<div class="version-grid"> <div class="version-grid">
{% for version in versions %} {% for minor in minors %}
<a href="{{ version }}/index.html" class="version-card"> <a href="{{ major }}.{{ minor }}/index.html" class="version-card{% if not loop.first %} small{% endif %}">
<div class="version-number">{{ version }}</div> <div class="version-number">{{ major }}.{{ minor }}</div>
{% if loop.first %}
<div class="version-label">Rocky Linux</div> <div class="version-label">Rocky Linux</div>
<div class="version-browse">Browse man pages →</div> <div class="version-browse">Browse man pages →</div>
{% endif %}
</a> </a>
{% endfor %} {% endfor %}
</div> </div>
{% endfor %}
</div> </div>
</div> </div>
{% endblock %} {% endblock %}