Compare commits
38 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ca498b6f3c | |||
| 3a4f098011 | |||
| 1d06240f5d | |||
| df3014fd0e | |||
| 85e4422b00 | |||
| b2ce978f84 | |||
| 76fc2e9deb | |||
| 87692de410 | |||
| c62c8a30e2 | |||
| 50345f166f | |||
| 316610e932 | |||
| b4ffdb6560 | |||
| 7032898a60 | |||
| 1ba181a084 | |||
| 4d7c526b0e | |||
| c494e55e35 | |||
| 24ad0e6861 | |||
| e356431b72 | |||
| 89404a2042 | |||
| c6dc7fe310 | |||
| 411abf8683 | |||
| c3fef46d6a | |||
| a0994a5c16 | |||
| 7bda4a1155 | |||
| 3f2ecebb08 | |||
| c9b59737b9 | |||
| 067965d983 | |||
| b9f6697100 | |||
| 106e680f11 | |||
| b1e987f1b1 | |||
| 13308d4b4b | |||
| 9e2943754f | |||
| b20a70b860 | |||
| b371431aa5 | |||
| c86868b11c | |||
| 2315422d4f | |||
| 29f299f984 | |||
| 8428dc9b1a |
+10
-31
@@ -25,39 +25,26 @@ on:
|
|||||||
jobs:
|
jobs:
|
||||||
build:
|
build:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container:
|
|
||||||
image: rockylinux:9
|
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Install system dependencies
|
- name: Build Docker image
|
||||||
run: |
|
run: |
|
||||||
dnf install -y \
|
docker build -t rocky-man:latest .
|
||||||
python3.11 \
|
|
||||||
python3.11-pip \
|
|
||||||
mandoc \
|
|
||||||
rpm-build \
|
|
||||||
dnf-plugins-core \
|
|
||||||
git
|
|
||||||
|
|
||||||
- name: Install UV
|
- name: Create output directories
|
||||||
run: |
|
run: |
|
||||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
mkdir -p ./html ./tmp
|
||||||
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
|
||||||
|
|
||||||
- name: Install Python dependencies
|
- name: Build man pages in container
|
||||||
run: |
|
run: |
|
||||||
uv pip install --system -e .
|
docker run --rm \
|
||||||
|
-v "$(pwd)/html:/data/html" \
|
||||||
- name: Build man pages
|
-v "$(pwd)/tmp:/data/tmp" \
|
||||||
run: |
|
rocky-man:latest \
|
||||||
python3.11 -m rocky_man.main \
|
--versions ${{ github.event.inputs.versions || '8.10 9.6 10.0' }} \
|
||||||
--versions ${{ github.event.inputs.versions || '8.10 9.5' }} \
|
|
||||||
--output-dir ./html \
|
|
||||||
--download-dir ./tmp/downloads \
|
|
||||||
--extract-dir ./tmp/extracts \
|
|
||||||
--verbose
|
--verbose
|
||||||
env:
|
env:
|
||||||
PYTHONUNBUFFERED: 1
|
PYTHONUNBUFFERED: 1
|
||||||
@@ -68,11 +55,3 @@ jobs:
|
|||||||
name: rocky-man-pages
|
name: rocky-man-pages
|
||||||
path: html/
|
path: html/
|
||||||
retention-days: 30
|
retention-days: 30
|
||||||
|
|
||||||
- name: Deploy to GitHub Pages
|
|
||||||
if: github.ref == 'refs/heads/main'
|
|
||||||
uses: peaceiris/actions-gh-pages@v3
|
|
||||||
with:
|
|
||||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
publish_dir: ./html
|
|
||||||
force_orphan: true
|
|
||||||
|
|||||||
Binary file not shown.
+1
-1
@@ -18,7 +18,7 @@ RUN dnf install -y epel-release \
|
|||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
# Copy project files
|
# Copy project files
|
||||||
COPY pyproject.toml README.md LICENSE THIRD-PARTY-LICENSES.md ./
|
COPY pyproject.toml README.md LICENSE ./
|
||||||
COPY src ./src
|
COPY src ./src
|
||||||
COPY templates ./templates
|
COPY templates ./templates
|
||||||
|
|
||||||
|
|||||||
Vendored
+159
@@ -0,0 +1,159 @@
|
|||||||
|
// Jenkinsfile for Rocky Man
|
||||||
|
pipeline {
|
||||||
|
agent {
|
||||||
|
kubernetes {
|
||||||
|
yaml """
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
jenkins: agent
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: docker
|
||||||
|
image: docker:24-dind
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
volumeMounts:
|
||||||
|
- name: docker-sock
|
||||||
|
mountPath: /var/run
|
||||||
|
command:
|
||||||
|
- dockerd-entrypoint.sh
|
||||||
|
- name: docker-cli
|
||||||
|
image: docker:24-cli
|
||||||
|
command:
|
||||||
|
- cat
|
||||||
|
tty: true
|
||||||
|
volumeMounts:
|
||||||
|
- name: docker-sock
|
||||||
|
mountPath: /var/run
|
||||||
|
volumes:
|
||||||
|
- name: docker-sock
|
||||||
|
emptyDir: {}
|
||||||
|
"""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
parameters {
|
||||||
|
string(
|
||||||
|
name: 'VERSIONS',
|
||||||
|
defaultValue: '8.10 9.8 10.2',
|
||||||
|
description: 'Rocky Linux versions to build (space-separated)'
|
||||||
|
)
|
||||||
|
string(
|
||||||
|
name: 'REPO_TYPES',
|
||||||
|
defaultValue: 'BaseOS AppStream',
|
||||||
|
description: 'Rocky Linux repos to build against (space-separated)'
|
||||||
|
)
|
||||||
|
string(
|
||||||
|
name: 'R2_BUCKET_NAME',
|
||||||
|
defaultValue: 'rockyman',
|
||||||
|
description: 'R2 bucket name for uploads'
|
||||||
|
)
|
||||||
|
string(
|
||||||
|
name: 'EXISTING_VERSIONS',
|
||||||
|
defaultValue: '',
|
||||||
|
description: 'Existing versions already built (space-separated)'
|
||||||
|
)
|
||||||
|
string(
|
||||||
|
name: 'PARALLEL_DOWNLOADS',
|
||||||
|
defaultValue: '5',
|
||||||
|
description: 'Number of parallel downloads'
|
||||||
|
)
|
||||||
|
string(
|
||||||
|
name: 'PARALLEL_CONVERSIONS',
|
||||||
|
defaultValue: '10',
|
||||||
|
description: 'Number of parallel conversions'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
options {
|
||||||
|
buildDiscarder(logRotator(numToKeepStr: '10'))
|
||||||
|
timeout(time: 2, unit: 'HOURS')
|
||||||
|
}
|
||||||
|
|
||||||
|
stages {
|
||||||
|
stage('Checkout') {
|
||||||
|
steps {
|
||||||
|
checkout scm
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stage('Build Docker Image') {
|
||||||
|
steps {
|
||||||
|
container('docker-cli') {
|
||||||
|
sh '''
|
||||||
|
until docker info > /dev/null 2>&1; do
|
||||||
|
echo "Waiting for Docker daemon..."
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
docker build -t rocky-man:${BUILD_NUMBER} .
|
||||||
|
docker tag rocky-man:${BUILD_NUMBER} rocky-man:latest
|
||||||
|
'''
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stage('Build Man Pages') {
|
||||||
|
steps {
|
||||||
|
container('docker-cli') {
|
||||||
|
sh '''
|
||||||
|
mkdir -p ./html ./tmp
|
||||||
|
|
||||||
|
docker run --rm \
|
||||||
|
-v "$(pwd)/html:/data/html" \
|
||||||
|
-v "$(pwd)/tmp:/data/tmp" \
|
||||||
|
rocky-man:${BUILD_NUMBER} \
|
||||||
|
--versions ${VERSIONS} \
|
||||||
|
--repo-types ${REPO_TYPES} \
|
||||||
|
--parallel-downloads ${PARALLEL_DOWNLOADS} \
|
||||||
|
--parallel-conversions ${PARALLEL_CONVERSIONS} \
|
||||||
|
--existing-versions ${EXISTING_VERSIONS}
|
||||||
|
'''
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stage('Upload to R2') {
|
||||||
|
when {
|
||||||
|
expression { return params.R2_BUCKET_NAME != "" }
|
||||||
|
}
|
||||||
|
steps {
|
||||||
|
container('docker-cli') {
|
||||||
|
withCredentials([
|
||||||
|
string(credentialsId: 'r2-account-id', variable: 'R2_ACCOUNT_ID'),
|
||||||
|
string(credentialsId: 'r2-access-key-id', variable: 'AWS_ACCESS_KEY_ID'),
|
||||||
|
string(credentialsId: 'r2-secret-access-key', variable: 'AWS_SECRET_ACCESS_KEY')
|
||||||
|
]) {
|
||||||
|
sh '''
|
||||||
|
docker run --rm \
|
||||||
|
-v "$(pwd)/html:/workspace/html" \
|
||||||
|
-e AWS_ACCESS_KEY_ID \
|
||||||
|
-e AWS_SECRET_ACCESS_KEY \
|
||||||
|
peakcom/s5cmd:latest \
|
||||||
|
--endpoint-url "https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com" \
|
||||||
|
sync --delete /workspace/html/ "s3://${R2_BUCKET_NAME}/"
|
||||||
|
'''
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
post {
|
||||||
|
success {
|
||||||
|
echo 'Build completed and uploaded to R2!'
|
||||||
|
}
|
||||||
|
failure {
|
||||||
|
echo 'Build failed!'
|
||||||
|
}
|
||||||
|
cleanup {
|
||||||
|
container('docker-cli') {
|
||||||
|
sh '''
|
||||||
|
docker rmi rocky-man:${BUILD_NUMBER} || true
|
||||||
|
docker rmi rocky-man:latest || true
|
||||||
|
'''
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
MIT License
|
MIT License
|
||||||
|
|
||||||
Copyright (c) 2024 Stephen Simpson
|
Copyright (c) 2025 Ctrl IQ, Inc.
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
|||||||
@@ -1,528 +1,187 @@
|
|||||||
# Rocky Man 📚
|
# 🚀 Rocky Man 🚀
|
||||||
|
|
||||||
**Rocky Man** is a comprehensive man page hosting solution for Rocky Linux, providing beautiful, searchable documentation for all packages in BaseOS and AppStream repositories across Rocky Linux 8, 9, and 10.
|
**Rocky Man** is a tool for generating searchable HTML documentation from Rocky Linux man pages across BaseOS and AppStream repositories for Rocky Linux 8, 9, and 10.
|
||||||
|
|
||||||
> **✨ This is a complete rewrite** with 60-80% faster performance, modern architecture, and production-ready features!
|
|
||||||
|
|
||||||
## 🎉 What's New in This Rewrite
|
|
||||||
|
|
||||||
This version is a **complete ground-up rebuild** with major improvements:
|
|
||||||
|
|
||||||
- 🚀 **60-80% faster** - Pre-filters packages using filelists.xml (downloads only ~800 packages instead of ~3000)
|
|
||||||
- 🏗️ **Modular architecture** - Clean separation into models, repo, processor, web, and utils
|
|
||||||
- 🎨 **Modern UI** - Beautiful dark theme with instant fuzzy search
|
|
||||||
- 🐳 **Container ready** - Multi-stage Dockerfile that works on any architecture
|
|
||||||
- ⚡ **Parallel processing** - Concurrent downloads and HTML conversions
|
|
||||||
- 🧹 **Smart cleanup** - Automatic cleanup of temporary files
|
|
||||||
- 📝 **Well documented** - Comprehensive docstrings and type hints throughout
|
|
||||||
- 🔒 **Thread safe** - Proper locking and resource management
|
|
||||||
- 🤖 **GitHub Actions** - Automated weekly builds and deployment
|
|
||||||
|
|
||||||
### Performance Comparison
|
|
||||||
|
|
||||||
| Metric | Old Version | New Version | Improvement |
|
|
||||||
|--------|-------------|-------------|-------------|
|
|
||||||
| Packages Downloaded | ~3000 | ~800 | 73% reduction |
|
|
||||||
| Processing Time | 2-3 hours | 30-45 minutes | 75% faster |
|
|
||||||
| Bandwidth Used | ~10 GB | ~2-3 GB | 80% reduction |
|
|
||||||
| Architecture | Single file | Modular (16 files) | Much cleaner |
|
|
||||||
| Thread Safety | ⚠️ Issues | ✅ Safe | Fixed |
|
|
||||||
| Cleanup | Manual | Automatic | Improved |
|
|
||||||
| UI Quality | Basic | Modern | Much better |
|
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
- ✨ **Fast & Efficient**: Uses filelists.xml to pre-filter packages with man pages (massive bandwidth savings)
|
- Uses filelists.xml to pre-filter packages with man pages
|
||||||
- 🔍 **Fuzzy Search**: Instant search across all man pages with Fuse.js
|
- Processes packages from BaseOS and AppStream repositories
|
||||||
- 🎨 **Modern UI**: Clean, responsive dark theme interface inspired by GitHub
|
- Runs in containers on x86_64, aarch64, and arm64 architectures
|
||||||
- 📦 **Complete Coverage**: All packages from BaseOS and AppStream repositories
|
- Configurable cleanup of temporary files
|
||||||
- 🐳 **Container Ready**: Architecture-independent Docker support (works on x86_64, aarch64, arm64, etc.)
|
- Concurrent downloads and conversions
|
||||||
- 🚀 **GitHub Actions**: Automated weekly builds and deployment to GitHub Pages
|
- Supports Rocky Linux 8, 9, and 10
|
||||||
- 🧹 **Smart Cleanup**: Automatic cleanup of temporary files (configurable)
|
|
||||||
- ⚡ **Parallel Processing**: Concurrent downloads and conversions for maximum speed
|
|
||||||
- 🌐 **Multi-version**: Support for Rocky Linux 8, 9, and 10 simultaneously
|
|
||||||
|
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
### Option 1: Docker (Recommended)
|
### Podman
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Build the image
|
# Build the image
|
||||||
docker build -t rocky-man .
|
docker build -t rocky-man .
|
||||||
|
|
||||||
# Generate man pages for Rocky Linux 9.6
|
# Generate for specific versions
|
||||||
docker run --rm -v $(pwd)/html:/data/html rocky-man --versions 9.6
|
podman run --rm -v $(pwd)/html:/data/html:Z rocky-man \
|
||||||
|
--versions 8.10 9.6 10.0
|
||||||
|
|
||||||
# Generate for multiple versions
|
# Keep downloaded RPMs for multiple builds
|
||||||
docker run --rm -v $(pwd)/html:/data/html rocky-man --versions 8.10 9.6 10.0
|
|
||||||
|
|
||||||
# With verbose logging
|
|
||||||
docker run --rm -v $(pwd)/html:/data/html rocky-man --versions 9.6 --verbose
|
|
||||||
|
|
||||||
# Keep downloaded RPMs (mount the download directory)
|
|
||||||
docker run --rm -it \
|
|
||||||
-v $(pwd)/html:/data/html \
|
|
||||||
-v $(pwd)/downloads:/data/tmp/downloads \
|
|
||||||
rocky-man --versions 9.6 --keep-rpms --verbose
|
|
||||||
```
|
|
||||||
|
|
||||||
### Option 2: Podman (Native Rocky Linux)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Build the image
|
|
||||||
podman build -t rocky-man .
|
|
||||||
|
|
||||||
# Run with podman (note the :Z flag for SELinux)
|
|
||||||
podman run --rm -v $(pwd)/html:/data/html:Z rocky-man --versions 9.6
|
|
||||||
|
|
||||||
# Interactive mode for debugging
|
|
||||||
podman run --rm -it -v $(pwd)/html:/data/html:Z rocky-man --versions 9.6 --verbose
|
|
||||||
|
|
||||||
# Keep downloaded RPMs (mount the download directory)
|
|
||||||
podman run --rm -it \
|
podman run --rm -it \
|
||||||
-v $(pwd)/html:/data/html:Z \
|
-v $(pwd)/html:/data/html:Z \
|
||||||
-v $(pwd)/downloads:/data/tmp/downloads:Z \
|
-v $(pwd)/downloads:/data/tmp/downloads:Z \
|
||||||
rocky-man --versions 9.6 --keep-rpms --verbose
|
rocky-man --versions 9.6 --keep-rpms --verbose
|
||||||
```
|
```
|
||||||
|
|
||||||
### Option 3: Docker Compose (Development)
|
### View the HTML Locally
|
||||||
|
|
||||||
|
Start a local web server to browse the generated documentation:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Build and run
|
python3 -m http.server -d ./html
|
||||||
docker-compose up
|
```
|
||||||
|
|
||||||
# The generated HTML will be in ./html/
|
Then open [http://127.0.0.1:8000](http://127.0.0.1:8000) in your browser.
|
||||||
# Preview at http://localhost:8080 (nginx container)
|
|
||||||
|
To use a different port:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 -m http.server 8080 -d ./html
|
||||||
```
|
```
|
||||||
|
|
||||||
### Directory Structure in Container
|
### Directory Structure in Container
|
||||||
|
|
||||||
When running in a container, rocky-man uses these directories inside `/data/`:
|
The container uses the following paths:
|
||||||
|
|
||||||
- `/data/html` - Generated HTML output (mount this to access results)
|
- `/data/html` - Generated HTML output
|
||||||
- `/data/tmp/downloads` - Downloaded RPM files (temporary)
|
- `/data/tmp/downloads` - Downloaded RPM files
|
||||||
- `/data/tmp/extracts` - Extracted man page files (temporary)
|
- `/data/tmp/extracts` - Extracted man page files
|
||||||
|
|
||||||
By default, RPMs and extracts are automatically cleaned up after processing. If you want to keep the RPMs (e.g., for debugging or multiple runs), mount the download directory and use `--keep-rpms`:
|
These paths are used by default and can be overridden with command-line arguments if needed.
|
||||||
|
|
||||||
|
### Local Development
|
||||||
|
|
||||||
|
**Important**: Rocky Man requires Rocky Linux because it uses the system's native `python3-dnf` module to interact with DNF repositories. This module cannot be installed via pip and must come from the Rocky Linux system packages.
|
||||||
|
|
||||||
|
#### Option 1: Run in a Rocky Linux Container (Recommended)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# This keeps RPMs on your host in ./downloads/
|
# Start a Rocky Linux container with your project mounted
|
||||||
podman run --rm -it \
|
podman run --rm -it -v $(pwd):/workspace:Z rockylinux/rockylinux:9 /bin/bash
|
||||||
-v $(pwd)/html:/data/html:Z \
|
|
||||||
-v $(pwd)/downloads:/data/tmp/downloads:Z \
|
|
||||||
rocky-man --versions 9.6 --keep-rpms
|
|
||||||
```
|
|
||||||
|
|
||||||
**Note**: Without mounting `/data/tmp/downloads`, the `--keep-rpms` flag will keep files inside the container, but they'll be lost when the container stops (especially with `--rm`).
|
# Inside the container, navigate to the project
|
||||||
|
cd /workspace
|
||||||
|
|
||||||
### Option 4: Local Development
|
# Install epel-release for mandoc
|
||||||
|
dnf install -y epel-release
|
||||||
|
|
||||||
#### Prerequisites
|
# Install system dependencies
|
||||||
|
|
||||||
- Python 3.9+
|
|
||||||
- pip (Python package manager)
|
|
||||||
- mandoc (man page converter)
|
|
||||||
- Rocky Linux system or container (for DNF)
|
|
||||||
|
|
||||||
#### Installation
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# On Rocky Linux, install system dependencies
|
|
||||||
dnf install -y python3 python3-pip python3-dnf mandoc rpm-build dnf-plugins-core
|
dnf install -y python3 python3-pip python3-dnf mandoc rpm-build dnf-plugins-core
|
||||||
|
|
||||||
# Install Python dependencies
|
# Install Python dependencies
|
||||||
pip3 install -e .
|
pip3 install -e .
|
||||||
|
|
||||||
|
# Run the tool
|
||||||
|
python3 -m rocky_man.main --versions 9.6 --output-dir ./html/
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Usage
|
#### Option 2: On a Native Rocky Linux System
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Generate man pages for Rocky 9.6
|
# Install epel-release for mandoc
|
||||||
python -m rocky_man.main --versions 9.6
|
dnf install -y epel-release
|
||||||
|
|
||||||
# Generate for multiple versions (default)
|
# Install system dependencies
|
||||||
python -m rocky_man.main --versions 8.10 9.6 10.0
|
dnf install -y python3 python3-pip python3-dnf mandoc rpm-build dnf-plugins-core
|
||||||
|
|
||||||
# Custom output directory
|
# Install Python dependencies
|
||||||
python -m rocky_man.main --output-dir /var/www/html/man --versions 9.6
|
pip3 install -e .
|
||||||
|
|
||||||
# Keep downloaded RPMs for debugging
|
# Run the tool
|
||||||
python -m rocky_man.main --keep-rpms --verbose
|
python3 -m rocky_man.main --versions 9.6 --output-dir ./html/
|
||||||
|
|
||||||
# Adjust parallelism for faster processing
|
|
||||||
python -m rocky_man.main --parallel-downloads 10 --parallel-conversions 20
|
|
||||||
|
|
||||||
# Use a different mirror
|
|
||||||
python -m rocky_man.main --mirror https://mirrors.example.com/
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
Rocky Man is organized into clean, modular components:
|
Rocky Man is organized into components:
|
||||||
|
|
||||||
```
|
```text
|
||||||
rocky-man/
|
rocky-man/
|
||||||
├── src/rocky_man/
|
├── src/rocky_man/
|
||||||
│ ├── models/ # Data models (Package, ManFile)
|
│ ├── models/ # Data models (Package, ManFile)
|
||||||
│ │ ├── package.py # RPM package representation
|
|
||||||
│ │ └── manfile.py # Man page file representation
|
|
||||||
│ ├── repo/ # Repository management
|
│ ├── repo/ # Repository management
|
||||||
│ │ ├── manager.py # DNF repository operations
|
|
||||||
│ │ └── contents.py # Filelists.xml parser (key optimization!)
|
|
||||||
│ ├── processor/ # Man page processing
|
│ ├── processor/ # Man page processing
|
||||||
│ │ ├── extractor.py # Extract man pages from RPMs
|
|
||||||
│ │ └── converter.py # Convert to HTML with mandoc
|
|
||||||
│ ├── web/ # Web page generation
|
│ ├── web/ # Web page generation
|
||||||
│ │ └── generator.py # HTML and search index generation
|
|
||||||
│ ├── utils/ # Utilities
|
│ ├── utils/ # Utilities
|
||||||
│ │ └── config.py # Configuration management
|
|
||||||
│ └── main.py # Main entry point and orchestration
|
│ └── main.py # Main entry point and orchestration
|
||||||
├── templates/ # Jinja2 templates
|
├── templates/ # Jinja2 templates
|
||||||
│ ├── base.html # Base template with modern styling
|
|
||||||
│ ├── index.html # Search page with Fuse.js
|
|
||||||
│ ├── manpage.html # Individual man page display
|
|
||||||
│ └── root.html # Multi-version landing page
|
|
||||||
├── Dockerfile # Multi-stage, arch-independent
|
├── Dockerfile # Multi-stage, arch-independent
|
||||||
├── docker-compose.yml # Development setup with nginx
|
|
||||||
├── .github/workflows/ # GitHub Actions automation
|
|
||||||
└── pyproject.toml # Python project configuration
|
└── pyproject.toml # Python project configuration
|
||||||
```
|
```
|
||||||
|
|
||||||
### How It Works
|
### How It Works
|
||||||
|
|
||||||
1. **Package Discovery** 🔍
|
1. **Package Discovery** - Parses repository metadata (`repodata/repomd.xml` and `filelists.xml.gz`) to identify packages containing files in `/usr/share/man/` directories
|
||||||
- Parse repository `filelists.xml` to identify packages with man pages
|
2. **Package Download** - Downloads identified RPM packages using DNF, with configurable parallel downloads (default: 5)
|
||||||
- This is the **key optimization** - we know what to download before downloading!
|
3. **Man Page Extraction** - Extracts man page files from RPMs using `rpm2cpio`, filtering by section and language based on configuration
|
||||||
|
4. **HTML Conversion** - Converts troff-formatted man pages to HTML using mandoc, with parallel processing (default: 10 workers)
|
||||||
2. **Smart Download** ⬇️
|
5. **Cross-Reference Linking** - Parses converted HTML to add hyperlinks between man page references (e.g., `bash(1)` becomes clickable)
|
||||||
- Download only packages containing man pages (60-80% reduction)
|
6. **Index Generation** - Creates search indexes (JSON/gzipped) and navigation pages using Jinja2 templates
|
||||||
- Parallel downloads for speed
|
7. **Cleanup** - Removes temporary files (RPMs and extracted content) unless `--keep-rpms` or `--keep-extracts` is specified
|
||||||
- Architecture-independent (man pages are the same across arches)
|
|
||||||
|
|
||||||
3. **Extraction** 📦
|
|
||||||
- Extract man page files from RPM packages
|
|
||||||
- Handle gzipped and plain text man pages
|
|
||||||
- Support for multiple languages
|
|
||||||
|
|
||||||
4. **Conversion** 🔄
|
|
||||||
- Convert troff format to HTML using mandoc
|
|
||||||
- Clean up HTML output
|
|
||||||
- Parallel processing for speed
|
|
||||||
|
|
||||||
5. **Web Generation** 🌐
|
|
||||||
- Wrap HTML in beautiful templates
|
|
||||||
- Generate search index with fuzzy search
|
|
||||||
- Create multi-version navigation
|
|
||||||
|
|
||||||
6. **Cleanup** 🧹
|
|
||||||
- Automatically remove temporary files (configurable)
|
|
||||||
- Keep only what you need
|
|
||||||
|
|
||||||
## Command Line Options
|
## Command Line Options
|
||||||
|
|
||||||
```
|
```bash
|
||||||
usage: rocky-man [-h] [--versions VERSIONS [VERSIONS ...]]
|
usage: main.py [-h] [--versions VERSIONS [VERSIONS ...]]
|
||||||
[--repo-types REPO_TYPES [REPO_TYPES ...]]
|
[--repo-types REPO_TYPES [REPO_TYPES ...]]
|
||||||
[--output-dir OUTPUT_DIR] [--download-dir DOWNLOAD_DIR]
|
[--output-dir OUTPUT_DIR] [--download-dir DOWNLOAD_DIR]
|
||||||
[--extract-dir EXTRACT_DIR] [--keep-rpms] [--keep-extracts]
|
[--extract-dir EXTRACT_DIR] [--keep-rpms] [--keep-extracts]
|
||||||
[--parallel-downloads N] [--parallel-conversions N]
|
[--parallel-downloads PARALLEL_DOWNLOADS]
|
||||||
[--mirror URL] [--template-dir DIR] [-v]
|
[--parallel-conversions PARALLEL_CONVERSIONS] [--mirror MIRROR]
|
||||||
|
[--vault] [--existing-versions [VERSION ...]]
|
||||||
|
[--template-dir TEMPLATE_DIR] [-v]
|
||||||
|
[--skip-sections [SKIP_SECTIONS ...]]
|
||||||
|
[--skip-packages [SKIP_PACKAGES ...]] [--skip-languages]
|
||||||
|
[--keep-languages] [--allow-all-sections]
|
||||||
|
|
||||||
Generate HTML documentation for Rocky Linux man pages
|
Generate HTML documentation for Rocky Linux man pages
|
||||||
|
|
||||||
Options:
|
optional arguments:
|
||||||
-h, --help Show this help message and exit
|
-h, --help show this help message and exit
|
||||||
|
|
||||||
--versions VERSIONS [VERSIONS ...]
|
--versions VERSIONS [VERSIONS ...]
|
||||||
Rocky Linux versions to process (default: 8.10 9.6 10.0)
|
Rocky Linux versions to process (default: 8.10 9.6 10.0)
|
||||||
|
|
||||||
--repo-types REPO_TYPES [REPO_TYPES ...]
|
--repo-types REPO_TYPES [REPO_TYPES ...]
|
||||||
Repository types to process (default: BaseOS AppStream)
|
Repository types to process (default: BaseOS AppStream)
|
||||||
|
|
||||||
--output-dir OUTPUT_DIR
|
--output-dir OUTPUT_DIR
|
||||||
HTML output directory (default: ./html)
|
Output directory for HTML files (default: /data/html)
|
||||||
|
|
||||||
--download-dir DOWNLOAD_DIR
|
--download-dir DOWNLOAD_DIR
|
||||||
Package download directory (default: ./tmp/downloads)
|
Directory for downloading packages (default: /data/tmp/downloads)
|
||||||
|
|
||||||
--extract-dir EXTRACT_DIR
|
--extract-dir EXTRACT_DIR
|
||||||
Extraction directory (default: ./tmp/extracts)
|
Directory for extracting man pages (default: /data/tmp/extracts)
|
||||||
|
|
||||||
--keep-rpms Keep downloaded RPM files after processing
|
--keep-rpms Keep downloaded RPM files after processing
|
||||||
|
|
||||||
--keep-extracts Keep extracted man files after processing
|
--keep-extracts Keep extracted man files after processing
|
||||||
|
--parallel-downloads PARALLEL_DOWNLOADS
|
||||||
--parallel-downloads N
|
|
||||||
Number of parallel downloads (default: 5)
|
Number of parallel downloads (default: 5)
|
||||||
|
--parallel-conversions PARALLEL_CONVERSIONS
|
||||||
--parallel-conversions N
|
|
||||||
Number of parallel HTML conversions (default: 10)
|
Number of parallel HTML conversions (default: 10)
|
||||||
|
--mirror MIRROR Rocky Linux mirror URL (default: http://dl.rockylinux.org/)
|
||||||
--mirror URL Rocky Linux mirror URL
|
--vault Use vault directory instead of pub (vault/rocky instead of pub/rocky)
|
||||||
(default: http://dl.rockylinux.org/)
|
--existing-versions [VERSION ...]
|
||||||
|
List of existing versions to include in root index (e.g., 8.10 9.7)
|
||||||
--template-dir DIR Custom template directory
|
--template-dir TEMPLATE_DIR
|
||||||
|
Template directory (default: ./templates)
|
||||||
-v, --verbose Enable verbose logging
|
-v, --verbose Enable verbose logging
|
||||||
|
--skip-sections [SKIP_SECTIONS ...]
|
||||||
|
Man sections to skip (default: 3 3p 3pm). Use empty list to skip none.
|
||||||
|
--skip-packages [SKIP_PACKAGES ...]
|
||||||
|
Package names to skip (default: lapack dpdk-devel gl-manpages). Use empty list to skip none.
|
||||||
|
--skip-languages Skip non-English man pages (default: enabled)
|
||||||
|
--keep-languages Keep all languages (disables --skip-languages)
|
||||||
|
--allow-all-sections Include all man sections (overrides --skip-sections)
|
||||||
```
|
```
|
||||||
|
|
||||||
### Examples
|
## Attribution
|
||||||
|
|
||||||
```bash
|
The man pages displayed in this documentation are sourced from Rocky Linux distribution packages. All man page content is copyrighted by their respective authors and distributed under the licenses specified within each man page.
|
||||||
# Quick test with one version
|
|
||||||
python -m rocky_man.main --versions 9.6
|
|
||||||
|
|
||||||
# Production build with all versions (default)
|
This tool generates HTML documentation from man pages contained in Rocky Linux packages but does not modify the content of the man pages themselves.
|
||||||
python -m rocky_man.main
|
|
||||||
|
|
||||||
# Fast build with more parallelism
|
|
||||||
python -m rocky_man.main --parallel-downloads 15 --parallel-conversions 30
|
|
||||||
|
|
||||||
# Keep files for debugging
|
|
||||||
python -m rocky_man.main --keep-rpms --keep-extracts --verbose
|
|
||||||
|
|
||||||
# Custom mirror (faster for your location)
|
|
||||||
python -m rocky_man.main --mirror https://mirror.usi.edu/pub/rocky/
|
|
||||||
|
|
||||||
# Only BaseOS (faster)
|
|
||||||
python -m rocky_man.main --repo-types BaseOS --versions 9.6
|
|
||||||
```
|
|
||||||
|
|
||||||
## GitHub Actions Integration
|
|
||||||
|
|
||||||
This project includes a **production-ready GitHub Actions workflow** that:
|
|
||||||
|
|
||||||
- ✅ Runs automatically every Sunday at midnight UTC
|
|
||||||
- ✅ Can be manually triggered with custom version selection
|
|
||||||
- ✅ Builds man pages in a Rocky Linux container
|
|
||||||
- ✅ Automatically deploys to GitHub Pages
|
|
||||||
- ✅ Artifacts available for download
|
|
||||||
|
|
||||||
### Setup Instructions
|
|
||||||
|
|
||||||
1. **Enable GitHub Pages**
|
|
||||||
- Go to your repository → Settings → Pages
|
|
||||||
- Set source to **"GitHub Actions"**
|
|
||||||
- Save
|
|
||||||
|
|
||||||
2. **Trigger the workflow**
|
|
||||||
- Go to Actions tab
|
|
||||||
- Select "Build Rocky Man Pages"
|
|
||||||
- Click "Run workflow"
|
|
||||||
- Choose versions (or use default)
|
|
||||||
|
|
||||||
3. **Access your site**
|
|
||||||
- Will be available at: `https://YOUR_USERNAME.github.io/rocky-man/`
|
|
||||||
- Updates automatically every week!
|
|
||||||
|
|
||||||
### Workflow File
|
|
||||||
|
|
||||||
Located at `.github/workflows/build.yml`, it:
|
|
||||||
- Uses Rocky Linux 9 container
|
|
||||||
- Installs all dependencies
|
|
||||||
- Runs the build
|
|
||||||
- Uploads artifacts
|
|
||||||
- Deploys to GitHub Pages
|
|
||||||
|
|
||||||
## What's Different from the Original
|
|
||||||
|
|
||||||
| Feature | Old Version | New Version |
|
|
||||||
|---------|-------------|-------------|
|
|
||||||
| **Architecture** | Single 400-line file | Modular, 16 files across 6 modules |
|
|
||||||
| **Package Filtering** | Downloads everything | Pre-filters with filelists.xml |
|
|
||||||
| **Performance** | 2-3 hours, ~10 GB | 30-45 min, ~2-3 GB |
|
|
||||||
| **UI** | Basic template | Modern GitHub-inspired design |
|
|
||||||
| **Search** | Simple filter | Fuzzy search with Fuse.js |
|
|
||||||
| **Container** | Basic Podman commands | Multi-stage Dockerfile + compose |
|
|
||||||
| **Thread Safety** | Global dict issues | Proper locking mechanisms |
|
|
||||||
| **Cleanup** | Method exists but unused | Automatic, configurable |
|
|
||||||
| **Documentation** | Minimal comments | Comprehensive docstrings |
|
|
||||||
| **Type Hints** | None | Throughout codebase |
|
|
||||||
| **Error Handling** | Basic try/catch | Comprehensive with logging |
|
|
||||||
| **CI/CD** | None | GitHub Actions ready |
|
|
||||||
| **Testing** | None | Ready for pytest integration |
|
|
||||||
| **Configuration** | Hardcoded | Config class with defaults |
|
|
||||||
|
|
||||||
## Project Structure Details
|
|
||||||
|
|
||||||
```
|
|
||||||
rocky-man/
|
|
||||||
├── src/rocky_man/ # Main source code
|
|
||||||
│ ├── __init__.py # Package initialization
|
|
||||||
│ ├── main.py # Entry point and orchestration (200 lines)
|
|
||||||
│ ├── models/ # Data models
|
|
||||||
│ │ ├── __init__.py
|
|
||||||
│ │ ├── package.py # Package model with properties
|
|
||||||
│ │ └── manfile.py # ManFile model with path parsing
|
|
||||||
│ ├── repo/ # Repository operations
|
|
||||||
│ │ ├── __init__.py
|
|
||||||
│ │ ├── manager.py # DNF integration, downloads
|
|
||||||
│ │ └── contents.py # Filelists parser (key optimization)
|
|
||||||
│ ├── processor/ # Processing pipeline
|
|
||||||
│ │ ├── __init__.py
|
|
||||||
│ │ ├── extractor.py # RPM extraction with rpmfile
|
|
||||||
│ │ └── converter.py # mandoc conversion wrapper
|
|
||||||
│ ├── web/ # Web generation
|
|
||||||
│ │ ├── __init__.py
|
|
||||||
│ │ └── generator.py # Template rendering, search index
|
|
||||||
│ └── utils/ # Utilities
|
|
||||||
│ ├── __init__.py
|
|
||||||
│ └── config.py # Configuration management
|
|
||||||
├── templates/ # Jinja2 templates
|
|
||||||
│ ├── base.html # Base layout (modern dark theme)
|
|
||||||
│ ├── index.html # Search page (Fuse.js integration)
|
|
||||||
│ ├── manpage.html # Man page display
|
|
||||||
│ └── root.html # Multi-version landing
|
|
||||||
├── old/ # Your original code (preserved)
|
|
||||||
│ ├── rocky_man.py
|
|
||||||
│ ├── rocky_man2.py
|
|
||||||
│ └── templates/
|
|
||||||
├── .github/
|
|
||||||
│ └── workflows/
|
|
||||||
│ └── build.yml # GitHub Actions workflow
|
|
||||||
├── Dockerfile # Multi-stage build
|
|
||||||
├── .dockerignore # Optimize Docker context
|
|
||||||
├── docker-compose.yml # Dev environment
|
|
||||||
├── pyproject.toml # Python project config
|
|
||||||
├── .gitignore # Updated for new structure
|
|
||||||
└── README.md # This file!
|
|
||||||
```
|
|
||||||
|
|
||||||
## Development
|
|
||||||
|
|
||||||
### Adding New Features
|
|
||||||
|
|
||||||
The modular design makes it easy to extend:
|
|
||||||
|
|
||||||
- **New repositories**: Add to `config.repo_types` in `utils/config.py`
|
|
||||||
- **Custom templates**: Use `--template-dir` flag or modify `templates/`
|
|
||||||
- **Additional metadata**: Extend `Package` or `ManFile` models
|
|
||||||
- **Alternative converters**: Implement new converter in `processor/`
|
|
||||||
- **Different outputs**: Add new generator in `web/`
|
|
||||||
|
|
||||||
### Running Tests
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Install dev dependencies
|
|
||||||
pip3 install -e ".[dev]"
|
|
||||||
|
|
||||||
# Run tests (when implemented)
|
|
||||||
pytest
|
|
||||||
|
|
||||||
# Type checking
|
|
||||||
mypy src/
|
|
||||||
|
|
||||||
# Linting
|
|
||||||
ruff check src/
|
|
||||||
```
|
|
||||||
|
|
||||||
### Development Workflow
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 1. Make changes to code
|
|
||||||
vim src/rocky_man/processor/converter.py
|
|
||||||
|
|
||||||
# 2. Test locally in container
|
|
||||||
podman run --rm -it -v $(pwd):/app rockylinux:9 /bin/bash
|
|
||||||
cd /app
|
|
||||||
python3 -m rocky_man.main --versions 9.6 --verbose
|
|
||||||
|
|
||||||
# 3. Build Docker image
|
|
||||||
docker build -t rocky-man .
|
|
||||||
|
|
||||||
# 4. Test Docker image
|
|
||||||
docker run --rm -v $(pwd)/html:/data/html rocky-man --versions 9.6
|
|
||||||
|
|
||||||
# 5. Preview output
|
|
||||||
docker-compose up nginx
|
|
||||||
# Visit http://localhost:8080
|
|
||||||
|
|
||||||
# 6. Commit and push
|
|
||||||
git add .
|
|
||||||
git commit -m "feat: your feature description"
|
|
||||||
git push
|
|
||||||
```
|
|
||||||
|
|
||||||
## Troubleshooting
|
|
||||||
|
|
||||||
### DNF Errors
|
|
||||||
|
|
||||||
**Problem**: `dnf` module not found or repository errors
|
|
||||||
|
|
||||||
**Solution**: Ensure you're running on Rocky Linux or in a Rocky Linux container:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Run in Rocky Linux container
|
|
||||||
podman run --rm -it -v $(pwd):/app rockylinux:9 /bin/bash
|
|
||||||
cd /app
|
|
||||||
|
|
||||||
# Install dependencies
|
|
||||||
dnf install -y python3 python3-dnf mandoc rpm-build dnf-plugins-core
|
|
||||||
|
|
||||||
# Run the script
|
|
||||||
python3 -m rocky_man.main --versions 9.6
|
|
||||||
```
|
|
||||||
|
|
||||||
### Mandoc Not Found
|
|
||||||
|
|
||||||
**Problem**: `mandoc: command not found`
|
|
||||||
|
|
||||||
**Solution**: Install mandoc:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
dnf install -y mandoc
|
|
||||||
```
|
|
||||||
|
|
||||||
### Permission Errors in Container
|
|
||||||
|
|
||||||
**Problem**: Cannot write to mounted volume
|
|
||||||
|
|
||||||
**Solution**: Use the `:Z` flag with podman for SELinux contexts:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
podman run --rm -v $(pwd)/html:/data/html:Z rocky-man
|
|
||||||
```
|
|
||||||
|
|
||||||
For Docker, ensure the volume path is absolute:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker run --rm -v "$(pwd)/html":/data/html rocky-man
|
|
||||||
```
|
|
||||||
|
|
||||||
### Out of Memory
|
|
||||||
|
|
||||||
**Problem**: Process killed due to memory
|
|
||||||
|
|
||||||
**Solution**: Reduce parallelism:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python -m rocky_man.main --parallel-downloads 2 --parallel-conversions 5
|
|
||||||
```
|
|
||||||
|
|
||||||
### Slow Downloads
|
|
||||||
|
|
||||||
**Problem**: Downloads are very slow
|
|
||||||
|
|
||||||
**Solution**: Use a closer mirror:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Find mirrors at: https://mirrors.rockylinux.org/mirrormanager/mirrors
|
|
||||||
python -m rocky_man.main --mirror https://mirror.example.com/rocky/
|
|
||||||
```
|
|
||||||
|
|
||||||
### UTF-8 Decode Errors
|
|
||||||
|
|
||||||
**Problem**: `'utf-8' codec can't decode byte...`
|
|
||||||
|
|
||||||
**Solution**: This is now handled with `errors='replace'` in the new version. The man page will still be processed with replacement characters for invalid UTF-8.
|
|
||||||
|
|
||||||
## Performance Tips
|
|
||||||
|
|
||||||
1. **Use closer mirrors** - Significant speed improvement for downloads
|
|
||||||
2. **Increase parallelism** - If you have bandwidth: `--parallel-downloads 15`
|
|
||||||
3. **Process one repo at a time** - Use `--repo-types BaseOS` first, then `--repo-types AppStream`
|
|
||||||
4. **Keep RPMs for re-runs** - Use `--keep-rpms` if testing
|
|
||||||
5. **Run in container** - More consistent performance
|
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
@@ -530,51 +189,16 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
|
|||||||
|
|
||||||
### Third-Party Software
|
### Third-Party Software
|
||||||
|
|
||||||
This project uses several open source components. See [THIRD-PARTY-LICENSES.md](THIRD-PARTY-LICENSES.md) for complete license information and attributions.
|
This project uses several open source components.
|
||||||
|
|
||||||
|
Key dependencies include:
|
||||||
|
|
||||||
|
- **mandoc** - Man page converter (ISC License)
|
||||||
|
- **python3-dnf** - DNF package manager Python bindings (GPL-2.0-or-later)
|
||||||
|
- **Fuse.js** - Client-side search (Apache 2.0)
|
||||||
|
- **Python packages**: requests, rpmfile, Jinja2, lxml, zstandard
|
||||||
|
- **Fonts**: Red Hat Display, Red Hat Text, JetBrains Mono (SIL OFL)
|
||||||
|
|
||||||
### Trademark Notice
|
### Trademark Notice
|
||||||
|
|
||||||
Rocky Linux™ is a trademark of the Rocky Enterprise Software Foundation (RESF). This project is not officially affiliated with or endorsed by RESF. All trademarks are the property of their respective owners. This project complies with RESF's trademark usage guidelines.
|
Rocky Linux is a trademark of the Rocky Enterprise Software Foundation (RESF). This project is not officially affiliated with or endorsed by RESF. All trademarks are the property of their respective owners. This project complies with RESF's trademark usage guidelines.
|
||||||
|
|
||||||
## Contributing
|
|
||||||
|
|
||||||
Contributions welcome! Please:
|
|
||||||
|
|
||||||
1. Fork the repository
|
|
||||||
2. Create a feature branch (`git checkout -b feature/amazing-feature`)
|
|
||||||
3. Make your changes with proper documentation
|
|
||||||
4. Test thoroughly
|
|
||||||
5. Commit with clear messages (`git commit -m 'feat: add amazing feature'`)
|
|
||||||
6. Push to your branch (`git push origin feature/amazing-feature`)
|
|
||||||
7. Open a Pull Request
|
|
||||||
|
|
||||||
## Acknowledgments
|
|
||||||
|
|
||||||
- Inspired by [debiman](https://github.com/Debian/debiman) for Debian
|
|
||||||
- Uses [mandoc](https://mandoc.bsd.lv/) for man page conversion
|
|
||||||
- Search powered by [Fuse.js](https://fusejs.io/)
|
|
||||||
- Modern UI design inspired by GitHub's dark theme
|
|
||||||
|
|
||||||
## Links
|
|
||||||
|
|
||||||
- [Rocky Linux](https://rockylinux.org/)
|
|
||||||
- [Man Page Format](https://man7.org/linux/man-pages/)
|
|
||||||
- [Mandoc Documentation](https://mandoc.bsd.lv/)
|
|
||||||
- [DNF Documentation](https://dnf.readthedocs.io/)
|
|
||||||
|
|
||||||
## Roadmap
|
|
||||||
|
|
||||||
- [ ] Add pytest test suite
|
|
||||||
- [ ] Implement incremental updates (checksum-based)
|
|
||||||
- [ ] Add support for localized man pages (es, fr, etc.)
|
|
||||||
- [ ] Create redirect system like debiman
|
|
||||||
- [ ] Add statistics page (most viewed, etc.)
|
|
||||||
- [ ] Implement RSS feed for updates
|
|
||||||
- [ ] Add support for Rocky Linux 10 (when released)
|
|
||||||
- [ ] Create sitemap.xml for SEO
|
|
||||||
- [ ] Add dark/light theme toggle
|
|
||||||
- [ ] Implement caching for faster rebuilds
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Made with ❤️ for the Rocky Linux community**
|
|
||||||
|
|||||||
+4
-4
@@ -9,11 +9,11 @@ authors = [
|
|||||||
]
|
]
|
||||||
requires-python = ">=3.9"
|
requires-python = ">=3.9"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"requests>=2.31.0",
|
"requests>=2.32.0",
|
||||||
"rpmfile>=2.0.0",
|
"rpmfile>=2.1.0",
|
||||||
"jinja2>=3.1.0",
|
"jinja2>=3.1.0",
|
||||||
"lxml>=5.0.0",
|
"lxml>=6.0.0",
|
||||||
"zstandard>=0.18.0",
|
"zstandard>=0.25.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
|
|||||||
+116
-110
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import logging
|
import logging
|
||||||
|
import re
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -16,16 +17,12 @@ def setup_logging(verbose: bool = False):
|
|||||||
level = logging.DEBUG if verbose else logging.INFO
|
level = logging.DEBUG if verbose else logging.INFO
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
level=level,
|
level=level,
|
||||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||||
datefmt='%Y-%m-%d %H:%M:%S'
|
datefmt="%Y-%m-%d %H:%M:%S",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def process_version(
|
def process_version(config: Config, version: str, template_dir: Path) -> bool:
|
||||||
config: Config,
|
|
||||||
version: str,
|
|
||||||
template_dir: Path
|
|
||||||
) -> bool:
|
|
||||||
"""Process a single Rocky Linux version.
|
"""Process a single Rocky Linux version.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -46,31 +43,22 @@ def process_version(
|
|||||||
|
|
||||||
all_man_files = []
|
all_man_files = []
|
||||||
|
|
||||||
# Process each repository type
|
|
||||||
for repo_type in config.repo_types:
|
for repo_type in config.repo_types:
|
||||||
logger.info(f"Processing {repo_type} repository")
|
logger.info(f"Processing {repo_type} repository")
|
||||||
|
|
||||||
# Use first available architecture (man pages are arch-independent)
|
|
||||||
arch = config.architectures[0]
|
arch = config.architectures[0]
|
||||||
|
|
||||||
# Get repository URL
|
|
||||||
repo_url = config.get_repo_url(version, repo_type, arch)
|
|
||||||
|
|
||||||
# Create cache dir for this repo
|
|
||||||
cache_dir = config.download_dir / f".cache/{version}/{repo_type}"
|
cache_dir = config.download_dir / f".cache/{version}/{repo_type}"
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Initialize repository manager
|
|
||||||
repo_manager = RepoManager(
|
repo_manager = RepoManager(
|
||||||
repo_url=repo_url,
|
config=config,
|
||||||
version=version,
|
version=version,
|
||||||
repo_type=repo_type,
|
repo_type=repo_type,
|
||||||
arch=arch,
|
arch=arch,
|
||||||
cache_dir=cache_dir,
|
cache_dir=cache_dir,
|
||||||
download_dir=version_download_dir
|
download_dir=version_download_dir,
|
||||||
)
|
)
|
||||||
|
|
||||||
# List packages (with man pages only)
|
|
||||||
packages = repo_manager.list_packages(with_manpages_only=True)
|
packages = repo_manager.list_packages(with_manpages_only=True)
|
||||||
|
|
||||||
if not packages:
|
if not packages:
|
||||||
@@ -79,40 +67,35 @@ def process_version(
|
|||||||
|
|
||||||
logger.info(f"Found {len(packages)} packages with man pages in {repo_type}")
|
logger.info(f"Found {len(packages)} packages with man pages in {repo_type}")
|
||||||
|
|
||||||
# Filter out packages that should be skipped
|
|
||||||
if config.skip_packages:
|
if config.skip_packages:
|
||||||
original_count = len(packages)
|
original_count = len(packages)
|
||||||
packages = [
|
packages = [
|
||||||
pkg for pkg in packages
|
pkg for pkg in packages if pkg.name not in config.skip_packages
|
||||||
if pkg.name not in config.skip_packages
|
|
||||||
]
|
]
|
||||||
filtered_count = original_count - len(packages)
|
filtered_count = original_count - len(packages)
|
||||||
if filtered_count > 0:
|
if filtered_count > 0:
|
||||||
logger.info(f"Filtered out {filtered_count} packages based on skip list")
|
logger.info(
|
||||||
|
f"Filtered out {filtered_count} packages based on skip list"
|
||||||
|
)
|
||||||
logger.info(f"Processing {len(packages)} packages")
|
logger.info(f"Processing {len(packages)} packages")
|
||||||
|
|
||||||
# Download packages
|
|
||||||
logger.info("Downloading packages...")
|
logger.info("Downloading packages...")
|
||||||
downloaded = repo_manager.download_packages(
|
downloaded = repo_manager.download_packages(
|
||||||
packages,
|
packages, max_workers=config.parallel_downloads
|
||||||
max_workers=config.parallel_downloads
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Extract man pages
|
|
||||||
logger.info("Extracting man pages...")
|
logger.info("Extracting man pages...")
|
||||||
extractor = ManPageExtractor(
|
extractor = ManPageExtractor(
|
||||||
version_extract_dir,
|
version_extract_dir,
|
||||||
skip_sections=config.skip_sections,
|
skip_sections=config.skip_sections,
|
||||||
skip_languages=config.skip_languages
|
skip_languages=config.skip_languages,
|
||||||
)
|
)
|
||||||
man_files = extractor.extract_from_packages(
|
man_files = extractor.extract_from_packages(
|
||||||
downloaded,
|
downloaded, max_workers=config.parallel_downloads
|
||||||
max_workers=config.parallel_downloads
|
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info(f"Extracted {len(man_files)} man pages")
|
logger.info(f"Extracted {len(man_files)} man pages")
|
||||||
|
|
||||||
# Read content for each man file
|
|
||||||
logger.info("Reading man page content...")
|
logger.info("Reading man page content...")
|
||||||
man_files_with_content = []
|
man_files_with_content = []
|
||||||
for man_file in man_files:
|
for man_file in man_files:
|
||||||
@@ -120,17 +103,14 @@ def process_version(
|
|||||||
if content:
|
if content:
|
||||||
man_files_with_content.append((man_file, content))
|
man_files_with_content.append((man_file, content))
|
||||||
|
|
||||||
# Convert to HTML
|
|
||||||
logger.info("Converting man pages to HTML...")
|
logger.info("Converting man pages to HTML...")
|
||||||
converter = ManPageConverter(version_output_dir)
|
converter = ManPageConverter(version_output_dir)
|
||||||
converted = converter.convert_many(
|
converted = converter.convert_many(
|
||||||
man_files_with_content,
|
man_files_with_content, max_workers=config.parallel_conversions
|
||||||
max_workers=config.parallel_conversions
|
|
||||||
)
|
)
|
||||||
|
|
||||||
all_man_files.extend(converted)
|
all_man_files.extend(converted)
|
||||||
|
|
||||||
# Cleanup if requested
|
|
||||||
if not config.keep_rpms:
|
if not config.keep_rpms:
|
||||||
logger.info("Cleaning up downloaded packages...")
|
logger.info("Cleaning up downloaded packages...")
|
||||||
for package in downloaded:
|
for package in downloaded:
|
||||||
@@ -149,169 +129,177 @@ def process_version(
|
|||||||
logger.error(f"No man pages were successfully processed for version {version}")
|
logger.error(f"No man pages were successfully processed for version {version}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# Link cross-references between man pages
|
|
||||||
logger.info("Linking cross-references...")
|
|
||||||
converter = ManPageConverter(version_output_dir)
|
|
||||||
converter.link_cross_references(all_man_files)
|
|
||||||
|
|
||||||
# Generate web pages
|
|
||||||
logger.info("Generating web pages...")
|
logger.info("Generating web pages...")
|
||||||
web_gen = WebGenerator(template_dir, config.output_dir)
|
web_gen = WebGenerator(template_dir, config.output_dir)
|
||||||
|
|
||||||
# Generate search index
|
|
||||||
search_index = web_gen.generate_search_index(all_man_files, version)
|
search_index = web_gen.generate_search_index(all_man_files, version)
|
||||||
web_gen.save_search_index(search_index, version)
|
web_gen.save_search_index(search_index, version)
|
||||||
|
|
||||||
# Generate index page
|
|
||||||
web_gen.generate_index(version, search_index)
|
web_gen.generate_index(version, search_index)
|
||||||
|
|
||||||
# Generate packages index page
|
|
||||||
web_gen.generate_packages_index(version, search_index)
|
web_gen.generate_packages_index(version, search_index)
|
||||||
|
|
||||||
# Wrap man pages in templates
|
for man_file in all_man_files:
|
||||||
|
if not man_file.html_path:
|
||||||
|
man_file.html_path = web_gen._get_manpage_path(man_file, version)
|
||||||
|
|
||||||
|
logger.info("Linking cross-references...")
|
||||||
|
converter.link_cross_references(all_man_files, version)
|
||||||
|
|
||||||
logger.info("Generating man page HTML...")
|
logger.info("Generating man page HTML...")
|
||||||
for man_file in all_man_files:
|
for man_file in all_man_files:
|
||||||
web_gen.generate_manpage_html(man_file, version)
|
web_gen.generate_manpage_html(man_file, version)
|
||||||
|
|
||||||
logger.info(f"Successfully processed {len(all_man_files)} man pages for Rocky Linux {version}")
|
logger.info(
|
||||||
|
f"Successfully processed {len(all_man_files)} man pages for Rocky Linux {version}"
|
||||||
|
)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
"""Main entry point."""
|
"""Main entry point."""
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description='Generate HTML documentation for Rocky Linux man pages'
|
description="Generate HTML documentation for Rocky Linux man pages"
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--versions',
|
"--versions",
|
||||||
nargs='+',
|
nargs="+",
|
||||||
default=['8.10', '9.6', '10.0'],
|
default=["8.10", "9.6", "10.0"],
|
||||||
help='Rocky Linux versions to process (default: 8.10 9.6 10.0)'
|
help="Rocky Linux versions to process (default: 8.10 9.6 10.0)",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--repo-types',
|
"--repo-types",
|
||||||
nargs='+',
|
nargs="+",
|
||||||
default=['BaseOS', 'AppStream'],
|
default=["BaseOS", "AppStream"],
|
||||||
help='Repository types to process (default: BaseOS AppStream)'
|
help="Repository types to process (default: BaseOS AppStream)",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--output-dir',
|
"--output-dir",
|
||||||
type=Path,
|
type=Path,
|
||||||
default=Path('./html'),
|
default=Path("/data/html"),
|
||||||
help='Output directory for HTML files (default: ./html)'
|
help="Output directory for HTML files (default: /data/html)",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--download-dir',
|
"--download-dir",
|
||||||
type=Path,
|
type=Path,
|
||||||
default=Path('./tmp/downloads'),
|
default=Path("/data/tmp/downloads"),
|
||||||
help='Directory for downloading packages (default: ./tmp/downloads)'
|
help="Directory for downloading packages (default: /data/tmp/downloads)",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--extract-dir',
|
"--extract-dir",
|
||||||
type=Path,
|
type=Path,
|
||||||
default=Path('./tmp/extracts'),
|
default=Path("/data/tmp/extracts"),
|
||||||
help='Directory for extracting man pages (default: ./tmp/extracts)'
|
help="Directory for extracting man pages (default: /data/tmp/extracts)",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--keep-rpms',
|
"--keep-rpms",
|
||||||
action='store_true',
|
action="store_true",
|
||||||
help='Keep downloaded RPM files after processing'
|
help="Keep downloaded RPM files after processing",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--keep-extracts',
|
"--keep-extracts",
|
||||||
action='store_true',
|
action="store_true",
|
||||||
help='Keep extracted man files after processing'
|
help="Keep extracted man files after processing",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--parallel-downloads',
|
"--parallel-downloads",
|
||||||
type=int,
|
type=int,
|
||||||
default=5,
|
default=5,
|
||||||
help='Number of parallel downloads (default: 5)'
|
help="Number of parallel downloads (default: 5)",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--parallel-conversions',
|
"--parallel-conversions",
|
||||||
type=int,
|
type=int,
|
||||||
default=10,
|
default=10,
|
||||||
help='Number of parallel HTML conversions (default: 10)'
|
help="Number of parallel HTML conversions (default: 10)",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--mirror',
|
"--mirror",
|
||||||
default='http://dl.rockylinux.org/',
|
default="http://dl.rockylinux.org/",
|
||||||
help='Rocky Linux mirror URL (default: http://dl.rockylinux.org/)'
|
help="Rocky Linux mirror URL (default: http://dl.rockylinux.org/)",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--template-dir',
|
"--vault",
|
||||||
|
action="store_true",
|
||||||
|
help="Use vault directory instead of pub (vault/rocky instead of pub/rocky)",
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--existing-versions",
|
||||||
|
nargs="*",
|
||||||
|
metavar="VERSION",
|
||||||
|
help="List of existing versions to include in root index (e.g., 8.10 9.7)",
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--template-dir",
|
||||||
type=Path,
|
type=Path,
|
||||||
default=Path(__file__).parent.parent.parent / 'templates',
|
default=Path(__file__).parent.parent.parent / "templates",
|
||||||
help='Template directory (default: ./templates)'
|
help="Template directory (default: ./templates)",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'-v', '--verbose',
|
"-v", "--verbose", action="store_true", help="Enable verbose logging"
|
||||||
action='store_true',
|
|
||||||
help='Enable verbose logging'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--skip-sections',
|
"--skip-sections",
|
||||||
nargs='*',
|
nargs="*",
|
||||||
default=None,
|
default=None,
|
||||||
help='Man sections to skip (default: 3 3p 3pm). Use empty list to skip none.'
|
help="Man sections to skip (default: 3 3p 3pm). Use empty list to skip none.",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--skip-packages',
|
"--skip-packages",
|
||||||
nargs='*',
|
nargs="*",
|
||||||
default=None,
|
default=None,
|
||||||
help='Package names to skip (default: lapack dpdk-devel gl-manpages). Use empty list to skip none.'
|
help="Package names to skip (default: lapack dpdk-devel gl-manpages). Use empty list to skip none.",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--skip-languages',
|
"--skip-languages",
|
||||||
action='store_true',
|
action="store_true",
|
||||||
default=None,
|
default=None,
|
||||||
help='Skip non-English man pages (default: enabled)'
|
help="Skip non-English man pages (default: enabled)",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--keep-languages',
|
"--keep-languages",
|
||||||
action='store_true',
|
action="store_true",
|
||||||
help='Keep all languages (disables --skip-languages)'
|
help="Keep all languages (disables --skip-languages)",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--allow-all-sections',
|
"--allow-all-sections",
|
||||||
action='store_true',
|
action="store_true",
|
||||||
help='Include all man sections (overrides --skip-sections)'
|
help="Include all man sections (overrides --skip-sections)",
|
||||||
)
|
)
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
# Setup logging
|
|
||||||
setup_logging(args.verbose)
|
setup_logging(args.verbose)
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Handle filtering options
|
skip_languages = True
|
||||||
skip_languages = True # default
|
|
||||||
if args.keep_languages:
|
if args.keep_languages:
|
||||||
skip_languages = False
|
skip_languages = False
|
||||||
elif args.skip_languages is not None:
|
elif args.skip_languages is not None:
|
||||||
skip_languages = args.skip_languages
|
skip_languages = args.skip_languages
|
||||||
|
|
||||||
# Create configuration
|
content_dir = "vault/rocky" if args.vault else "pub/rocky"
|
||||||
|
|
||||||
config = Config(
|
config = Config(
|
||||||
base_url=args.mirror,
|
base_url=args.mirror,
|
||||||
|
content_dir=content_dir,
|
||||||
versions=args.versions,
|
versions=args.versions,
|
||||||
repo_types=args.repo_types,
|
repo_types=args.repo_types,
|
||||||
download_dir=args.download_dir,
|
download_dir=args.download_dir,
|
||||||
@@ -324,15 +312,32 @@ def main():
|
|||||||
skip_sections=args.skip_sections,
|
skip_sections=args.skip_sections,
|
||||||
skip_packages=args.skip_packages,
|
skip_packages=args.skip_packages,
|
||||||
skip_languages=skip_languages,
|
skip_languages=skip_languages,
|
||||||
allow_all_sections=args.allow_all_sections
|
allow_all_sections=args.allow_all_sections,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
scanned_versions = [
|
||||||
|
d.name
|
||||||
|
for d in config.output_dir.iterdir()
|
||||||
|
if d.is_dir() and re.match(r"\d+\.\d+", d.name)
|
||||||
|
]
|
||||||
|
arg_versions = args.existing_versions or []
|
||||||
|
|
||||||
|
def version_key(v):
|
||||||
|
try:
|
||||||
|
major, minor = v.split(".")
|
||||||
|
return (int(major), int(minor))
|
||||||
|
except (ValueError, AttributeError):
|
||||||
|
return (0, 0)
|
||||||
|
|
||||||
|
existing_versions = sorted(set(scanned_versions + arg_versions), key=version_key)
|
||||||
|
all_versions = sorted(set(existing_versions + config.versions), key=version_key)
|
||||||
|
|
||||||
logger.info("Rocky Man - Rocky Linux Man Page Generator")
|
logger.info("Rocky Man - Rocky Linux Man Page Generator")
|
||||||
logger.info(f"Versions: {', '.join(config.versions)}")
|
logger.info(f"Versions to process: {', '.join(config.versions)}")
|
||||||
|
logger.info(f"All known versions: {', '.join(all_versions)}")
|
||||||
logger.info(f"Repositories: {', '.join(config.repo_types)}")
|
logger.info(f"Repositories: {', '.join(config.repo_types)}")
|
||||||
logger.info(f"Output directory: {config.output_dir}")
|
logger.info(f"Output directory: {config.output_dir}")
|
||||||
|
|
||||||
# Log filtering configuration
|
|
||||||
if config.skip_sections:
|
if config.skip_sections:
|
||||||
logger.info(f"Skipping man sections: {', '.join(config.skip_sections)}")
|
logger.info(f"Skipping man sections: {', '.join(config.skip_sections)}")
|
||||||
else:
|
else:
|
||||||
@@ -346,7 +351,6 @@ def main():
|
|||||||
else:
|
else:
|
||||||
logger.info("Including all languages")
|
logger.info("Including all languages")
|
||||||
|
|
||||||
# Process each version
|
|
||||||
processed_versions = []
|
processed_versions = []
|
||||||
for version in config.versions:
|
for version in config.versions:
|
||||||
try:
|
try:
|
||||||
@@ -359,10 +363,12 @@ def main():
|
|||||||
logger.error("No versions were successfully processed")
|
logger.error("No versions were successfully processed")
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
# Generate root index
|
|
||||||
logger.info("Generating root index page...")
|
logger.info("Generating root index page...")
|
||||||
web_gen = WebGenerator(args.template_dir, config.output_dir)
|
web_gen = WebGenerator(args.template_dir, config.output_dir)
|
||||||
web_gen.generate_root_index(processed_versions)
|
web_gen.generate_root_index(all_versions)
|
||||||
|
|
||||||
|
logger.info("Generating 404 page...")
|
||||||
|
web_gen.generate_404_page()
|
||||||
|
|
||||||
logger.info("=" * 60)
|
logger.info("=" * 60)
|
||||||
logger.info("Processing complete!")
|
logger.info("Processing complete!")
|
||||||
@@ -373,5 +379,5 @@ def main():
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == "__main__":
|
||||||
sys.exit(main())
|
sys.exit(main())
|
||||||
|
|||||||
@@ -35,35 +35,22 @@ class ManFile:
|
|||||||
self._parse_path()
|
self._parse_path()
|
||||||
|
|
||||||
def _parse_path(self):
|
def _parse_path(self):
|
||||||
"""Extract section, name, and language from the file path.
|
"""Extract section, name, and language from the file path."""
|
||||||
|
|
||||||
Example paths:
|
|
||||||
/usr/share/man/man1/bash.1.gz
|
|
||||||
/usr/share/man/es/man1/bash.1.gz
|
|
||||||
/usr/share/man/man3/printf.3.gz
|
|
||||||
"""
|
|
||||||
parts = self.file_path.parts
|
parts = self.file_path.parts
|
||||||
filename = self.file_path.name
|
filename = self.file_path.name
|
||||||
|
|
||||||
# Remove .gz extension if present
|
|
||||||
if filename.endswith('.gz'):
|
if filename.endswith('.gz'):
|
||||||
filename = filename[:-3]
|
filename = filename[:-3]
|
||||||
|
|
||||||
# Extract section from parent directory (e.g., 'man1', 'man3p', 'man3pm')
|
|
||||||
for part in reversed(parts):
|
for part in reversed(parts):
|
||||||
if part.startswith('man') and len(part) > 3:
|
if part.startswith('man') and len(part) > 3:
|
||||||
# Check if it starts with 'man' followed by a digit
|
|
||||||
if part[3].isdigit():
|
if part[3].isdigit():
|
||||||
self.section = part[3:]
|
self.section = part[3:]
|
||||||
break
|
break
|
||||||
|
|
||||||
# Extract section from filename if not found yet (e.g., 'foo.3pm' -> section '3pm')
|
|
||||||
# and extract name
|
|
||||||
name_parts = filename.split('.')
|
name_parts = filename.split('.')
|
||||||
if len(name_parts) >= 2:
|
if len(name_parts) >= 2:
|
||||||
# Try to identify section from last part
|
|
||||||
potential_section = name_parts[-1]
|
potential_section = name_parts[-1]
|
||||||
# Section is typically digit optionally followed by letters (1, 3p, 3pm, etc.)
|
|
||||||
if potential_section and potential_section[0].isdigit():
|
if potential_section and potential_section[0].isdigit():
|
||||||
if not self.section:
|
if not self.section:
|
||||||
self.section = potential_section
|
self.section = potential_section
|
||||||
@@ -73,14 +60,10 @@ class ManFile:
|
|||||||
else:
|
else:
|
||||||
self.name = name_parts[0]
|
self.name = name_parts[0]
|
||||||
|
|
||||||
# Check for language subdirectory
|
|
||||||
# Pattern: /usr/share/man/<lang>/man<section>/
|
|
||||||
for i, part in enumerate(parts):
|
for i, part in enumerate(parts):
|
||||||
if part == 'man' and i + 1 < len(parts):
|
if part == 'man' and i + 1 < len(parts):
|
||||||
next_part = parts[i + 1]
|
next_part = parts[i + 1]
|
||||||
# If next part is not 'man<digit>', it's a language code
|
|
||||||
if not (next_part.startswith('man') and next_part[3:].isdigit()):
|
if not (next_part.startswith('man') and next_part[3:].isdigit()):
|
||||||
# Common language codes are 2-5 chars (en, es, pt_BR, etc.)
|
|
||||||
if len(next_part) <= 5:
|
if len(next_part) <= 5:
|
||||||
self.language = next_part
|
self.language = next_part
|
||||||
break
|
break
|
||||||
@@ -93,14 +76,12 @@ class ManFile:
|
|||||||
@property
|
@property
|
||||||
def html_filename(self) -> str:
|
def html_filename(self) -> str:
|
||||||
"""Get the HTML filename for this man page."""
|
"""Get the HTML filename for this man page."""
|
||||||
# Clean name for filesystem safety
|
|
||||||
safe_name = self._clean_filename(self.name)
|
safe_name = self._clean_filename(self.name)
|
||||||
suffix = f".{self.language}" if self.language else ""
|
suffix = f".{self.language}" if self.language else ""
|
||||||
return f"{safe_name}.{self.section}{suffix}.html"
|
return f"{safe_name}.{self.section}{suffix}.html"
|
||||||
|
|
||||||
def _clean_filename(self, name: str) -> str:
|
def _clean_filename(self, name: str) -> str:
|
||||||
"""Clean filename for filesystem safety."""
|
"""Clean filename for filesystem safety."""
|
||||||
# Replace problematic characters
|
|
||||||
name = name.replace('/', '_')
|
name = name.replace('/', '_')
|
||||||
name = name.replace(':', '_')
|
name = name.replace(':', '_')
|
||||||
name = re.sub(r'\.\.', '__', name)
|
name = re.sub(r'\.\.', '__', name)
|
||||||
@@ -108,19 +89,13 @@ class ManFile:
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def uri_path(self) -> str:
|
def uri_path(self) -> str:
|
||||||
"""Get the URI path for this man page (relative to version root).
|
"""Get the URI path for this man page (relative to version root)."""
|
||||||
|
|
||||||
Returns path like: 'bash/man1/bash.1.html'
|
|
||||||
"""
|
|
||||||
if not self.html_path:
|
if not self.html_path:
|
||||||
return ""
|
return ""
|
||||||
# Get path relative to the version directory
|
|
||||||
# Assuming structure: html/<version>/<package>/<section>/<file>.html
|
|
||||||
parts = self.html_path.parts
|
parts = self.html_path.parts
|
||||||
try:
|
try:
|
||||||
# Find the version part (e.g., '9.5') and return everything after it
|
|
||||||
for i, part in enumerate(parts):
|
for i, part in enumerate(parts):
|
||||||
if re.match(r'\d+\.\d+', part): # Version pattern
|
if re.match(r'\d+\.\d+', part):
|
||||||
return '/'.join(parts[i+1:])
|
return '/'.join(parts[i+1:])
|
||||||
except (ValueError, IndexError):
|
except (ValueError, IndexError):
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -38,19 +38,11 @@ class ManPageConverter:
|
|||||||
def _check_mandoc() -> bool:
|
def _check_mandoc() -> bool:
|
||||||
"""Check if mandoc is available."""
|
"""Check if mandoc is available."""
|
||||||
try:
|
try:
|
||||||
# Run mandoc with no arguments - it will show usage and exit
|
subprocess.run(["mandoc"], capture_output=True, timeout=5)
|
||||||
# We just want to verify the command exists, not that it succeeds
|
|
||||||
subprocess.run(
|
|
||||||
['mandoc'],
|
|
||||||
capture_output=True,
|
|
||||||
timeout=5
|
|
||||||
)
|
|
||||||
return True
|
return True
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
# mandoc command not found
|
|
||||||
return False
|
return False
|
||||||
except Exception:
|
except Exception:
|
||||||
# Other errors (timeout, etc) - but mandoc exists
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def convert(self, man_file: ManFile, content: str) -> bool:
|
def convert(self, man_file: ManFile, content: str) -> bool:
|
||||||
@@ -64,25 +56,39 @@ class ManPageConverter:
|
|||||||
True if conversion successful
|
True if conversion successful
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
# Run mandoc to convert to HTML
|
|
||||||
html = self._run_mandoc(content)
|
html = self._run_mandoc(content)
|
||||||
if not html:
|
if not html:
|
||||||
logger.warning(f"mandoc produced no output for {man_file.display_name}")
|
logger.warning(f"mandoc produced no output for {man_file.display_name}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# Clean up HTML
|
|
||||||
html = self._clean_html(html)
|
html = self._clean_html(html)
|
||||||
|
|
||||||
# Store in ManFile object
|
# Check if output indicates this is a symlink/redirect
|
||||||
man_file.html_content = html
|
symlink_match = re.search(
|
||||||
|
r'<div class="manual-text">.*?(?:See the file )?((?:/usr/share/man/)?man\d+[a-z]*/([^/]+)\.(\d+[a-z]*)(?:\.gz)?)\..*?</div>',
|
||||||
|
html,
|
||||||
|
re.DOTALL,
|
||||||
|
)
|
||||||
|
if not symlink_match:
|
||||||
|
symlink_match = re.search(
|
||||||
|
r'<div class="manual-text">.*?((?:/usr/share/man/)?man\d+[a-z]*/([^/<]+)\.(\d+[a-z]*)(?:\.gz)?).*?</div>',
|
||||||
|
html,
|
||||||
|
re.DOTALL,
|
||||||
|
)
|
||||||
|
|
||||||
# Determine output path
|
if symlink_match:
|
||||||
|
name = symlink_match.group(2)
|
||||||
|
section = symlink_match.group(3)
|
||||||
|
logger.info(
|
||||||
|
f"{man_file.display_name} detected as symlink to {name}({section})"
|
||||||
|
)
|
||||||
|
html = self._generate_redirect_html({"name": name, "section": section})
|
||||||
|
|
||||||
|
man_file.html_content = html
|
||||||
output_path = self._get_output_path(man_file)
|
output_path = self._get_output_path(man_file)
|
||||||
man_file.html_path = output_path
|
man_file.html_path = output_path
|
||||||
|
|
||||||
# Save HTML file
|
|
||||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
with open(output_path, 'w', encoding='utf-8') as f:
|
with open(output_path, "w", encoding="utf-8") as f:
|
||||||
f.write(html)
|
f.write(html)
|
||||||
|
|
||||||
logger.debug(f"Converted {man_file.display_name} -> {output_path}")
|
logger.debug(f"Converted {man_file.display_name} -> {output_path}")
|
||||||
@@ -93,9 +99,7 @@ class ManPageConverter:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
def convert_many(
|
def convert_many(
|
||||||
self,
|
self, man_files: List[tuple], max_workers: int = 10
|
||||||
man_files: List[tuple],
|
|
||||||
max_workers: int = 10
|
|
||||||
) -> List[ManFile]:
|
) -> List[ManFile]:
|
||||||
"""Convert multiple man pages in parallel.
|
"""Convert multiple man pages in parallel.
|
||||||
|
|
||||||
@@ -109,13 +113,11 @@ class ManPageConverter:
|
|||||||
converted = []
|
converted = []
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||||
# Submit all conversion tasks
|
|
||||||
future_to_manfile = {
|
future_to_manfile = {
|
||||||
executor.submit(self.convert, man_file, content): man_file
|
executor.submit(self.convert, man_file, content): man_file
|
||||||
for man_file, content in man_files
|
for man_file, content in man_files
|
||||||
}
|
}
|
||||||
|
|
||||||
# Collect results
|
|
||||||
for future in as_completed(future_to_manfile):
|
for future in as_completed(future_to_manfile):
|
||||||
man_file = future_to_manfile[future]
|
man_file = future_to_manfile[future]
|
||||||
try:
|
try:
|
||||||
@@ -138,21 +140,20 @@ class ManPageConverter:
|
|||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
['mandoc', '-T', 'html', '-O', 'fragment,toc'],
|
["mandoc", "-T", "html", "-O", "fragment,toc"],
|
||||||
input=content.encode('utf-8'),
|
input=content.encode("utf-8"),
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
timeout=30
|
timeout=30,
|
||||||
)
|
)
|
||||||
|
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
stderr = result.stderr.decode('utf-8', errors='replace')
|
stderr = result.stderr.decode("utf-8", errors="replace")
|
||||||
logger.warning(f"mandoc returned error: {stderr}")
|
logger.warning(f"mandoc returned error: {stderr}")
|
||||||
# Sometimes mandoc returns non-zero but still produces output
|
|
||||||
if result.stdout:
|
if result.stdout:
|
||||||
return result.stdout.decode('utf-8', errors='replace')
|
return result.stdout.decode("utf-8", errors="replace")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
return result.stdout.decode('utf-8', errors='replace')
|
return result.stdout.decode("utf-8", errors="replace")
|
||||||
|
|
||||||
except subprocess.TimeoutExpired:
|
except subprocess.TimeoutExpired:
|
||||||
logger.error("mandoc conversion timed out")
|
logger.error("mandoc conversion timed out")
|
||||||
@@ -170,123 +171,113 @@ class ManPageConverter:
|
|||||||
Returns:
|
Returns:
|
||||||
Cleaned HTML
|
Cleaned HTML
|
||||||
"""
|
"""
|
||||||
# Remove empty parentheses in header cells
|
# Fix empty header cells
|
||||||
html = re.sub(
|
html = re.sub(
|
||||||
r'<td class="head-ltitle">\(\)</td>',
|
r'<td class="head-(ltitle|rtitle)">\(\)</td>',
|
||||||
'<td class="head-ltitle"></td>',
|
r'<td class="head-\1"></td>',
|
||||||
html
|
html,
|
||||||
)
|
|
||||||
html = re.sub(
|
|
||||||
r'<td class="head-rtitle">\(\)</td>',
|
|
||||||
'<td class="head-rtitle"></td>',
|
|
||||||
html
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Strip leading/trailing whitespace
|
# Remove empty <p class="Pp"></p> tags (from .sp directives in troff)
|
||||||
|
html = re.sub(r'<p class="Pp">\s*</p>', '', html)
|
||||||
|
|
||||||
|
# Clean up trailing whitespace and br tags in pre blocks
|
||||||
|
# Match: <pre>...</pre> and clean trailing <br/> followed by whitespace
|
||||||
|
def clean_pre_block(match):
|
||||||
|
content = match.group(1)
|
||||||
|
# Remove trailing <br/> tags and whitespace before closing </pre>
|
||||||
|
content = re.sub(r'<br\s*/>\s*$', '', content)
|
||||||
|
content = re.sub(r'\s+$', '', content)
|
||||||
|
return f'<pre>{content}</pre>'
|
||||||
|
|
||||||
|
html = re.sub(r'<pre>(.*?)</pre>', clean_pre_block, html, flags=re.DOTALL)
|
||||||
|
|
||||||
html = html.strip()
|
html = html.strip()
|
||||||
|
|
||||||
return html
|
return html
|
||||||
|
|
||||||
def link_cross_references(self, man_files: List[ManFile]) -> None:
|
def _generate_redirect_html(self, target_info: dict) -> str:
|
||||||
"""Add hyperlinks to cross-references in SEE ALSO sections.
|
"""Generate HTML for a symlink/redirect page.
|
||||||
|
|
||||||
Goes through all converted HTML files and converts man page references
|
Args:
|
||||||
like pty(4) into working hyperlinks.
|
target_info: Dict with 'name' and 'section' of target man page
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
HTML fragment for redirect page
|
||||||
|
"""
|
||||||
|
name = target_info["name"]
|
||||||
|
section = target_info["section"]
|
||||||
|
target_filename = f"{name}.{section}.html"
|
||||||
|
|
||||||
|
html = f'''<div class="symlink-notice" style="padding: 2rem; text-align: center; background-color: var(--bg-tertiary); border-radius: 8px; border: 1px solid var(--border-color);">
|
||||||
|
<p style="font-size: 1.2rem; margin-bottom: 1.5rem; color: var(--text-primary);">
|
||||||
|
This is an alias for <b>{name}</b>({section}).
|
||||||
|
</p>
|
||||||
|
<p style="font-size: 1.1rem;">
|
||||||
|
<a href="{target_filename}" style="color: var(--accent-primary); text-decoration: none; font-weight: 500;">View the manual page</a>
|
||||||
|
</p>
|
||||||
|
</div>'''
|
||||||
|
return html
|
||||||
|
|
||||||
|
def link_cross_references(self, man_files: List[ManFile], version: str) -> None:
|
||||||
|
"""Add hyperlinks to cross-references in man pages.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
man_files: List of all converted ManFile objects
|
man_files: List of all converted ManFile objects
|
||||||
|
version: Rocky Linux version
|
||||||
"""
|
"""
|
||||||
# Build lookup index: (name, section) -> relative_path
|
|
||||||
lookup = {}
|
lookup = {}
|
||||||
for mf in man_files:
|
for mf in man_files:
|
||||||
key = (mf.name.lower(), str(mf.section))
|
key = (mf.name.lower(), str(mf.section))
|
||||||
if key not in lookup:
|
if key not in lookup:
|
||||||
# Store the relative path from the version root
|
|
||||||
lookup[key] = f"{mf.package_name}/man{mf.section}/{mf.html_filename}"
|
lookup[key] = f"{mf.package_name}/man{mf.section}/{mf.html_filename}"
|
||||||
|
|
||||||
logger.info(f"Linking cross-references across {len(man_files)} man pages...")
|
logger.info(f"Linking cross-references across {len(man_files)} man pages...")
|
||||||
|
|
||||||
# Process each man page HTML file
|
|
||||||
for man_file in man_files:
|
for man_file in man_files:
|
||||||
if not man_file.html_path or not man_file.html_path.exists():
|
if not man_file.html_content:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Read the HTML
|
html = man_file.html_content
|
||||||
with open(man_file.html_path, 'r', encoding='utf-8') as f:
|
pattern = (
|
||||||
html = f.read()
|
r"<b>([\w\-_.]+)</b>\((\d+[a-z]*)\)|\b([\w\-_.]+)\((\d+[a-z]*)\)"
|
||||||
|
)
|
||||||
# Find and replace man page references
|
|
||||||
# Mandoc outputs references as: <b>name</b>(section)
|
|
||||||
# Pattern matches both <b>name</b>(section) and plain name(section)
|
|
||||||
pattern = r'<b>([\w\-_.]+)</b>\((\d+[a-z]*)\)|\b([\w\-_.]+)\((\d+[a-z]*)\)'
|
|
||||||
|
|
||||||
def replace_reference(match):
|
def replace_reference(match):
|
||||||
full_match = match.group(0)
|
full_match = match.group(0)
|
||||||
|
|
||||||
# Check if this match is already inside an <a> tag
|
# Skip if already inside an <a> tag
|
||||||
# Look back up to 500 chars for context
|
|
||||||
before_text = html[max(0, match.start() - 500) : match.start()]
|
before_text = html[max(0, match.start() - 500) : match.start()]
|
||||||
|
last_open = before_text.rfind("<a ")
|
||||||
# Find the last <a and last </a> before this match
|
last_close = before_text.rfind("</a>")
|
||||||
last_open = before_text.rfind('<a ')
|
|
||||||
last_close = before_text.rfind('</a>')
|
|
||||||
|
|
||||||
# If the last <a> is after the last </a>, we're inside a link
|
|
||||||
if last_open > last_close:
|
if last_open > last_close:
|
||||||
return full_match
|
return full_match
|
||||||
|
|
||||||
if match.group(1): # <b>name</b>(section) format
|
name = (match.group(1) or match.group(3)).lower()
|
||||||
name = match.group(1).lower()
|
section = match.group(2) or match.group(4)
|
||||||
section = match.group(2)
|
|
||||||
else: # plain name(section) format
|
|
||||||
name = match.group(3).lower()
|
|
||||||
section = match.group(4)
|
|
||||||
|
|
||||||
# Look up the referenced man page
|
|
||||||
key = (name, section)
|
key = (name, section)
|
||||||
if key in lookup:
|
if key in lookup:
|
||||||
# Calculate relative path from current file to target
|
|
||||||
target_path = lookup[key]
|
target_path = lookup[key]
|
||||||
# File structure: output_dir/version/package_name/manN/file.html
|
rel_path = f"../../../{version}/{target_path}"
|
||||||
# Need to go up 3 levels to reach version root
|
|
||||||
# Current: package_name/manN/file.html
|
|
||||||
# Target: other_package/manM/file.html
|
|
||||||
rel_path = f"../../../{target_path}"
|
|
||||||
return f'<a href="{rel_path}">{full_match}</a>'
|
return f'<a href="{rel_path}">{full_match}</a>'
|
||||||
|
|
||||||
return full_match
|
return full_match
|
||||||
|
|
||||||
updated_html = re.sub(pattern, replace_reference, html)
|
updated_html = re.sub(pattern, replace_reference, html)
|
||||||
|
|
||||||
# Only write if something changed
|
|
||||||
if updated_html != html:
|
if updated_html != html:
|
||||||
with open(man_file.html_path, 'w', encoding='utf-8') as f:
|
man_file.html_content = updated_html
|
||||||
f.write(updated_html)
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Error linking references in {man_file.display_name}: {e}")
|
logger.warning(
|
||||||
|
f"Error linking references in {man_file.display_name}: {e}"
|
||||||
|
)
|
||||||
|
|
||||||
logger.info("Cross-reference linking complete")
|
logger.info("Cross-reference linking complete")
|
||||||
|
|
||||||
def _get_output_path(self, man_file: ManFile) -> Path:
|
def _get_output_path(self, man_file: ManFile) -> Path:
|
||||||
"""Determine output path for HTML file.
|
"""Determine output path for HTML file."""
|
||||||
|
|
||||||
Structure: output_dir/<package>/<section>/<name>.<section>[.<lang>].html
|
|
||||||
|
|
||||||
Args:
|
|
||||||
man_file: ManFile object
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Path for HTML output
|
|
||||||
"""
|
|
||||||
# Package directory
|
|
||||||
pkg_dir = self.output_dir / man_file.package_name
|
pkg_dir = self.output_dir / man_file.package_name
|
||||||
|
|
||||||
# Section directory (man1, man2, etc.)
|
|
||||||
section_dir = pkg_dir / f"man{man_file.section}"
|
section_dir = pkg_dir / f"man{man_file.section}"
|
||||||
|
return section_dir / man_file.html_filename
|
||||||
# HTML filename
|
|
||||||
filename = man_file.html_filename
|
|
||||||
|
|
||||||
return section_dir / filename
|
|
||||||
|
|||||||
@@ -48,7 +48,6 @@ class ManPageExtractor:
|
|||||||
logger.warning(f"Package file not found: {package.name}")
|
logger.warning(f"Package file not found: {package.name}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
# Create extraction directory for this package
|
|
||||||
pkg_extract_dir = self.extract_dir / package.name
|
pkg_extract_dir = self.extract_dir / package.name
|
||||||
pkg_extract_dir.mkdir(parents=True, exist_ok=True)
|
pkg_extract_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
@@ -59,33 +58,39 @@ class ManPageExtractor:
|
|||||||
|
|
||||||
with rpmfile.open(package.download_path) as rpm:
|
with rpmfile.open(package.download_path) as rpm:
|
||||||
for member in rpm.getmembers():
|
for member in rpm.getmembers():
|
||||||
# Check if this is a man page file
|
|
||||||
if not self._is_manpage(member.name):
|
if not self._is_manpage(member.name):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Create ManFile object
|
# Sanitize path to prevent path traversal attacks
|
||||||
extract_path = pkg_extract_dir / member.name.lstrip('/')
|
safe_name = member.name.lstrip('/')
|
||||||
|
extract_path = pkg_extract_dir / safe_name
|
||||||
|
|
||||||
|
# Resolve to absolute path and verify it's within the extraction directory
|
||||||
|
real_extract_path = extract_path.resolve()
|
||||||
|
real_pkg_extract_dir = pkg_extract_dir.resolve()
|
||||||
|
|
||||||
|
if not real_extract_path.is_relative_to(real_pkg_extract_dir):
|
||||||
|
logger.warning(f"Skipping file with path traversal attempt: {member.name}")
|
||||||
|
continue
|
||||||
|
|
||||||
man_file = ManFile(
|
man_file = ManFile(
|
||||||
file_path=extract_path,
|
file_path=real_extract_path,
|
||||||
package_name=package.name
|
package_name=package.name
|
||||||
)
|
)
|
||||||
|
|
||||||
# Apply section filtering
|
|
||||||
if self.skip_sections and man_file.section in self.skip_sections:
|
if self.skip_sections and man_file.section in self.skip_sections:
|
||||||
logger.debug(f"Skipping {man_file.display_name} (section {man_file.section})")
|
logger.debug(f"Skipping {man_file.display_name} (section {man_file.section})")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Apply language filtering
|
|
||||||
if self.skip_languages and man_file.language and man_file.language != 'en':
|
if self.skip_languages and man_file.language and man_file.language != 'en':
|
||||||
logger.debug(f"Skipping {man_file.display_name} (language {man_file.language})")
|
logger.debug(f"Skipping {man_file.display_name} (language {man_file.language})")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Extract the file
|
real_extract_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
extract_path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
content = rpm.extractfile(member).read()
|
content = rpm.extractfile(member).read()
|
||||||
with open(extract_path, 'wb') as f:
|
with open(real_extract_path, 'wb') as f:
|
||||||
f.write(content)
|
f.write(content)
|
||||||
|
|
||||||
man_file.content = content
|
man_file.content = content
|
||||||
@@ -118,13 +123,11 @@ class ManPageExtractor:
|
|||||||
all_man_files = []
|
all_man_files = []
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||||
# Submit all extraction tasks
|
|
||||||
future_to_pkg = {
|
future_to_pkg = {
|
||||||
executor.submit(self.extract_from_package, pkg): pkg
|
executor.submit(self.extract_from_package, pkg): pkg
|
||||||
for pkg in packages
|
for pkg in packages
|
||||||
}
|
}
|
||||||
|
|
||||||
# Collect results
|
|
||||||
for future in as_completed(future_to_pkg):
|
for future in as_completed(future_to_pkg):
|
||||||
pkg = future_to_pkg[future]
|
pkg = future_to_pkg[future]
|
||||||
try:
|
try:
|
||||||
@@ -150,27 +153,15 @@ class ManPageExtractor:
|
|||||||
return ""
|
return ""
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Try reading as gzipped file first
|
|
||||||
if man_file.file_path.suffix == '.gz':
|
if man_file.file_path.suffix == '.gz':
|
||||||
with gzip.open(man_file.file_path, 'rb') as f:
|
|
||||||
content = f.read()
|
|
||||||
else:
|
|
||||||
# Read as plain text
|
|
||||||
with open(man_file.file_path, 'rb') as f:
|
|
||||||
content = f.read()
|
|
||||||
|
|
||||||
# Decode with error handling
|
|
||||||
return content.decode('utf-8', errors='replace')
|
|
||||||
|
|
||||||
except gzip.BadGzipFile:
|
|
||||||
# Not a gzip file, try reading as plain text
|
|
||||||
try:
|
try:
|
||||||
|
with gzip.open(man_file.file_path, 'rb') as f:
|
||||||
|
return f.read().decode('utf-8', errors='replace')
|
||||||
|
except gzip.BadGzipFile:
|
||||||
|
pass
|
||||||
|
|
||||||
with open(man_file.file_path, 'rb') as f:
|
with open(man_file.file_path, 'rb') as f:
|
||||||
content = f.read()
|
return f.read().decode('utf-8', errors='replace')
|
||||||
return content.decode('utf-8', errors='replace')
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error reading {man_file.file_path}: {e}")
|
|
||||||
return ""
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error reading {man_file.file_path}: {e}")
|
logger.error(f"Error reading {man_file.file_path}: {e}")
|
||||||
@@ -178,37 +169,19 @@ class ManPageExtractor:
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _is_manpage(path: str) -> bool:
|
def _is_manpage(path: str) -> bool:
|
||||||
"""Check if a file path is a man page.
|
"""Check if a file path is a man page."""
|
||||||
|
|
||||||
Args:
|
|
||||||
path: File path to check
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
True if this looks like a man page file
|
|
||||||
"""
|
|
||||||
# Must contain /man/ in path
|
|
||||||
if '/man/' not in path:
|
if '/man/' not in path:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# Should be in /usr/share/man/ or /usr/man/
|
|
||||||
if not ('/share/man/' in path or path.startswith('/usr/man/')):
|
if not ('/share/man/' in path or path.startswith('/usr/man/')):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# Common man page patterns
|
|
||||||
# - /usr/share/man/man1/foo.1.gz
|
|
||||||
# - /usr/share/man/es/man1/foo.1.gz
|
|
||||||
# - /usr/share/man/man3/printf.3.gz
|
|
||||||
|
|
||||||
parts = path.split('/')
|
parts = path.split('/')
|
||||||
|
return any(
|
||||||
# Check for man<digit> directory
|
|
||||||
has_man_section = any(
|
|
||||||
part.startswith('man') and len(part) > 3 and part[3].isdigit()
|
part.startswith('man') and len(part) > 3 and part[3].isdigit()
|
||||||
for part in parts
|
for part in parts
|
||||||
)
|
)
|
||||||
|
|
||||||
return has_man_section
|
|
||||||
|
|
||||||
def cleanup_extracts(self, package: Package):
|
def cleanup_extracts(self, package: Package):
|
||||||
"""Clean up extracted files for a package.
|
"""Clean up extracted files for a package.
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import gzip
|
|||||||
import logging
|
import logging
|
||||||
import xml.etree.ElementTree as ET
|
import xml.etree.ElementTree as ET
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Set, Dict
|
from typing import Set
|
||||||
from urllib.parse import urljoin
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
@@ -38,19 +38,16 @@ class ContentsParser:
|
|||||||
"""
|
"""
|
||||||
logger.info(f"Fetching filelists for {self.repo_url}")
|
logger.info(f"Fetching filelists for {self.repo_url}")
|
||||||
|
|
||||||
# Download and parse repomd.xml to find filelists location
|
|
||||||
filelists_path = self._get_filelists_path()
|
filelists_path = self._get_filelists_path()
|
||||||
if not filelists_path:
|
if not filelists_path:
|
||||||
logger.warning("Could not find filelists in repository metadata")
|
logger.warning("Could not find filelists in repository metadata")
|
||||||
return set()
|
return set()
|
||||||
|
|
||||||
# Download filelists.xml
|
|
||||||
filelists_file = self._download_filelists(filelists_path)
|
filelists_file = self._download_filelists(filelists_path)
|
||||||
if not filelists_file:
|
if not filelists_file:
|
||||||
logger.warning("Could not download filelists")
|
logger.warning("Could not download filelists")
|
||||||
return set()
|
return set()
|
||||||
|
|
||||||
# Parse filelists to find packages with man pages
|
|
||||||
packages = self._parse_filelists(filelists_file)
|
packages = self._parse_filelists(filelists_file)
|
||||||
logger.info(f"Found {len(packages)} packages with man pages")
|
logger.info(f"Found {len(packages)} packages with man pages")
|
||||||
|
|
||||||
@@ -68,11 +65,7 @@ class ContentsParser:
|
|||||||
response = requests.get(repomd_url, timeout=30)
|
response = requests.get(repomd_url, timeout=30)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
|
|
||||||
# Parse XML
|
|
||||||
root = ET.fromstring(response.content)
|
root = ET.fromstring(response.content)
|
||||||
|
|
||||||
# Find filelists entry
|
|
||||||
# XML structure: <repomd><data type="filelists"><location href="..."/></data></repomd>
|
|
||||||
ns = {'repo': 'http://linux.duke.edu/metadata/repo'}
|
ns = {'repo': 'http://linux.duke.edu/metadata/repo'}
|
||||||
|
|
||||||
for data in root.findall('repo:data', ns):
|
for data in root.findall('repo:data', ns):
|
||||||
@@ -81,7 +74,7 @@ class ContentsParser:
|
|||||||
if location is not None:
|
if location is not None:
|
||||||
return location.get('href')
|
return location.get('href')
|
||||||
|
|
||||||
# Fallback: try without namespace
|
# Fallback without namespace
|
||||||
for data in root.findall('data'):
|
for data in root.findall('data'):
|
||||||
if data.get('type') == 'filelists':
|
if data.get('type') == 'filelists':
|
||||||
location = data.find('location')
|
location = data.find('location')
|
||||||
@@ -105,7 +98,6 @@ class ContentsParser:
|
|||||||
url = urljoin(self.repo_url, relative_path)
|
url = urljoin(self.repo_url, relative_path)
|
||||||
cache_file = self.cache_dir / relative_path.split('/')[-1]
|
cache_file = self.cache_dir / relative_path.split('/')[-1]
|
||||||
|
|
||||||
# Return cached file if it exists
|
|
||||||
if cache_file.exists():
|
if cache_file.exists():
|
||||||
logger.debug(f"Using cached filelists: {cache_file}")
|
logger.debug(f"Using cached filelists: {cache_file}")
|
||||||
return cache_file
|
return cache_file
|
||||||
@@ -138,36 +130,26 @@ class ContentsParser:
|
|||||||
packages = set()
|
packages = set()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Open gzipped XML file
|
|
||||||
with gzip.open(filelists_path, 'rb') as f:
|
with gzip.open(filelists_path, 'rb') as f:
|
||||||
# Use iterparse for memory efficiency (files can be large)
|
|
||||||
context = ET.iterparse(f, events=('start', 'end'))
|
context = ET.iterparse(f, events=('start', 'end'))
|
||||||
|
|
||||||
current_package = None
|
current_package = None
|
||||||
has_manpage = False
|
has_manpage = False
|
||||||
|
|
||||||
for event, elem in context:
|
for event, elem in context:
|
||||||
if event == 'start':
|
if event == 'start' and elem.tag.endswith('package'):
|
||||||
if elem.tag.endswith('package'):
|
|
||||||
# Get package name from 'name' attribute
|
|
||||||
current_package = elem.get('name')
|
current_package = elem.get('name')
|
||||||
has_manpage = False
|
has_manpage = False
|
||||||
|
|
||||||
elif event == 'end':
|
elif event == 'end':
|
||||||
if elem.tag.endswith('file'):
|
if elem.tag.endswith('file'):
|
||||||
# Check if file path contains /man/
|
|
||||||
file_path = elem.text
|
file_path = elem.text
|
||||||
if file_path and '/man/' in file_path:
|
if file_path and self._is_manpage_path(file_path):
|
||||||
# Could be /usr/share/man/ or /usr/man/
|
|
||||||
if '/share/man/' in file_path or file_path.startswith('/usr/man/'):
|
|
||||||
has_manpage = True
|
has_manpage = True
|
||||||
|
|
||||||
elif elem.tag.endswith('package'):
|
elif elem.tag.endswith('package'):
|
||||||
# End of package entry
|
|
||||||
if has_manpage and current_package:
|
if has_manpage and current_package:
|
||||||
packages.add(current_package)
|
packages.add(current_package)
|
||||||
|
|
||||||
# Clear element to free memory
|
|
||||||
elem.clear()
|
elem.clear()
|
||||||
current_package = None
|
current_package = None
|
||||||
has_manpage = False
|
has_manpage = False
|
||||||
@@ -177,45 +159,16 @@ class ContentsParser:
|
|||||||
|
|
||||||
return packages
|
return packages
|
||||||
|
|
||||||
def get_package_man_files(self, filelists_path: Path) -> Dict[str, list]:
|
@staticmethod
|
||||||
"""Get detailed list of man files for each package.
|
def _is_manpage_path(file_path: str) -> bool:
|
||||||
|
"""Check if a file path is a man page location.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
filelists_path: Path to filelists.xml.gz file
|
file_path: File path to check
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dict mapping package name to list of man page paths
|
True if path is in a standard man page directory
|
||||||
"""
|
"""
|
||||||
packages = {}
|
return '/man/' in file_path and (
|
||||||
|
'/share/man/' in file_path or file_path.startswith('/usr/man/')
|
||||||
try:
|
)
|
||||||
with gzip.open(filelists_path, 'rb') as f:
|
|
||||||
context = ET.iterparse(f, events=('start', 'end'))
|
|
||||||
|
|
||||||
current_package = None
|
|
||||||
current_files = []
|
|
||||||
|
|
||||||
for event, elem in context:
|
|
||||||
if event == 'start':
|
|
||||||
if elem.tag.endswith('package'):
|
|
||||||
current_package = elem.get('name')
|
|
||||||
current_files = []
|
|
||||||
|
|
||||||
elif event == 'end':
|
|
||||||
if elem.tag.endswith('file'):
|
|
||||||
file_path = elem.text
|
|
||||||
if file_path and '/share/man/' in file_path:
|
|
||||||
current_files.append(file_path)
|
|
||||||
|
|
||||||
elif elem.tag.endswith('package'):
|
|
||||||
if current_files and current_package:
|
|
||||||
packages[current_package] = current_files
|
|
||||||
|
|
||||||
elem.clear()
|
|
||||||
current_package = None
|
|
||||||
current_files = []
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error parsing filelists: {e}")
|
|
||||||
|
|
||||||
return packages
|
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ class RepoManager:
|
|||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
repo_url: str,
|
config,
|
||||||
version: str,
|
version: str,
|
||||||
repo_type: str,
|
repo_type: str,
|
||||||
arch: str,
|
arch: str,
|
||||||
@@ -35,14 +35,14 @@ class RepoManager:
|
|||||||
"""Initialize repository manager.
|
"""Initialize repository manager.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
repo_url: Full repository URL
|
config: Configuration object
|
||||||
version: Rocky Linux version (e.g., '9.5')
|
version: Rocky Linux version (e.g., '9.5')
|
||||||
repo_type: Repository type ('BaseOS' or 'AppStream')
|
repo_type: Repository type ('BaseOS' or 'AppStream')
|
||||||
arch: Architecture (e.g., 'x86_64')
|
arch: Architecture (e.g., 'x86_64')
|
||||||
cache_dir: Directory for caching metadata
|
cache_dir: Directory for caching metadata
|
||||||
download_dir: Directory for downloading packages
|
download_dir: Directory for downloading packages
|
||||||
"""
|
"""
|
||||||
self.repo_url = repo_url
|
self.config = config
|
||||||
self.version = version
|
self.version = version
|
||||||
self.repo_type = repo_type
|
self.repo_type = repo_type
|
||||||
self.arch = arch
|
self.arch = arch
|
||||||
@@ -52,13 +52,12 @@ class RepoManager:
|
|||||||
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
||||||
self.download_dir.mkdir(parents=True, exist_ok=True)
|
self.download_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
# Initialize DNF
|
|
||||||
self.base = dnf.Base()
|
self.base = dnf.Base()
|
||||||
self.base.conf.debuglevel = 0
|
self.base.conf.debuglevel = 0
|
||||||
self.base.conf.errorlevel = 0
|
self.base.conf.errorlevel = 0
|
||||||
self.base.conf.cachedir = str(self.cache_dir / "dnf")
|
self.base.conf.cachedir = str(self.cache_dir / "dnf")
|
||||||
|
|
||||||
self._configure_repo()
|
self.repo_url = None
|
||||||
self.packages_with_manpages: Optional[Set[str]] = None
|
self.packages_with_manpages: Optional[Set[str]] = None
|
||||||
|
|
||||||
def _configure_repo(self):
|
def _configure_repo(self):
|
||||||
@@ -67,29 +66,48 @@ class RepoManager:
|
|||||||
repo = dnf.repo.Repo(repo_id, self.base.conf)
|
repo = dnf.repo.Repo(repo_id, self.base.conf)
|
||||||
repo.baseurl = [self.repo_url]
|
repo.baseurl = [self.repo_url]
|
||||||
repo.enabled = True
|
repo.enabled = True
|
||||||
repo.gpgcheck = False # We verify checksums separately
|
repo.gpgcheck = False
|
||||||
|
|
||||||
self.base.repos.add(repo)
|
self.base.repos.add(repo)
|
||||||
logger.info(f"Configured repository: {repo_id} at {self.repo_url}")
|
logger.info(f"Configured repository: {repo_id} at {self.repo_url}")
|
||||||
|
|
||||||
# Fill the sack (package database)
|
|
||||||
self.base.fill_sack(load_system_repo=False, load_available_repos=True)
|
self.base.fill_sack(load_system_repo=False, load_available_repos=True)
|
||||||
logger.info("Repository metadata loaded")
|
logger.info("Repository metadata loaded")
|
||||||
|
|
||||||
def discover_packages_with_manpages(self) -> Set[str]:
|
def discover_packages_with_manpages(self) -> Set[str]:
|
||||||
"""Discover which packages contain man pages using filelists.
|
"""Discover which packages contain man pages using filelists.
|
||||||
|
|
||||||
This is the key optimization - we parse repository metadata
|
|
||||||
to identify packages with man pages before downloading anything.
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Set of package names that contain man pages
|
Set of package names that contain man pages
|
||||||
"""
|
"""
|
||||||
if self.packages_with_manpages is not None:
|
if self.packages_with_manpages is not None:
|
||||||
return self.packages_with_manpages
|
return self.packages_with_manpages
|
||||||
|
|
||||||
parser = ContentsParser(self.repo_url, self.cache_dir)
|
content_dirs = ["pub/rocky", "vault/rocky"]
|
||||||
self.packages_with_manpages = parser.get_packages_with_manpages()
|
for content_dir in content_dirs:
|
||||||
|
original_content_dir = self.config.content_dir
|
||||||
|
self.config.content_dir = content_dir
|
||||||
|
try:
|
||||||
|
repo_url = self.config.get_repo_url(
|
||||||
|
self.version, self.repo_type, self.arch
|
||||||
|
)
|
||||||
|
parser = ContentsParser(repo_url, self.cache_dir)
|
||||||
|
packages = parser.get_packages_with_manpages()
|
||||||
|
if packages:
|
||||||
|
self.packages_with_manpages = packages
|
||||||
|
self.repo_url = repo_url
|
||||||
|
logger.info(f"Using repository: {repo_url}")
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
logger.warning(f"No man pages found in {content_dir}, trying next")
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Failed to load metadata from {content_dir}: {e}")
|
||||||
|
finally:
|
||||||
|
self.config.content_dir = original_content_dir
|
||||||
|
else:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Failed to load repository metadata for {self.version} {self.repo_type} from both pub and vault"
|
||||||
|
)
|
||||||
|
|
||||||
return self.packages_with_manpages
|
return self.packages_with_manpages
|
||||||
|
|
||||||
@@ -102,38 +120,33 @@ class RepoManager:
|
|||||||
Returns:
|
Returns:
|
||||||
List of Package objects
|
List of Package objects
|
||||||
"""
|
"""
|
||||||
logger.info(f"Querying packages from {self.repo_type} ({self.version}/{self.arch})")
|
logger.info(
|
||||||
|
f"Querying packages from {self.repo_type} ({self.version}/{self.arch})"
|
||||||
|
)
|
||||||
|
|
||||||
# Get packages with man pages if filtering
|
|
||||||
manpage_packages = None
|
manpage_packages = None
|
||||||
if with_manpages_only:
|
if with_manpages_only:
|
||||||
manpage_packages = self.discover_packages_with_manpages()
|
manpage_packages = self.discover_packages_with_manpages()
|
||||||
logger.info(f"Filtering to {len(manpage_packages)} packages with man pages")
|
logger.info(f"Filtering to {len(manpage_packages)} packages with man pages")
|
||||||
|
|
||||||
|
self._configure_repo()
|
||||||
|
|
||||||
packages = []
|
packages = []
|
||||||
|
|
||||||
# Query all available packages
|
|
||||||
query = self.base.sack.query().available()
|
query = self.base.sack.query().available()
|
||||||
|
|
||||||
# For each package name, get only one arch (prefer noarch, then our target arch)
|
|
||||||
seen_names = set()
|
seen_names = set()
|
||||||
|
|
||||||
for pkg in query:
|
for pkg in query:
|
||||||
pkg_name = pkg.name
|
pkg_name = pkg.name
|
||||||
|
|
||||||
# Skip if we've already added this package
|
|
||||||
if pkg_name in seen_names:
|
if pkg_name in seen_names:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Skip if filtering and package doesn't have man pages
|
|
||||||
if manpage_packages and pkg_name not in manpage_packages:
|
if manpage_packages and pkg_name not in manpage_packages:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Get repo information
|
|
||||||
repo = pkg.repo
|
repo = pkg.repo
|
||||||
baseurl = repo.baseurl[0] if repo and repo.baseurl else self.repo_url
|
baseurl = repo.baseurl[0] if repo and repo.baseurl else self.repo_url
|
||||||
|
chksum_type, chksum_value = pkg.chksum if pkg.chksum else ("sha256", "")
|
||||||
# Create Package object
|
|
||||||
package = Package(
|
package = Package(
|
||||||
name=pkg_name,
|
name=pkg_name,
|
||||||
version=pkg.version,
|
version=pkg.version,
|
||||||
@@ -142,16 +155,16 @@ class RepoManager:
|
|||||||
repo_type=self.repo_type,
|
repo_type=self.repo_type,
|
||||||
location=pkg.location,
|
location=pkg.location,
|
||||||
baseurl=baseurl,
|
baseurl=baseurl,
|
||||||
checksum=pkg.chksum[1] if pkg.chksum else "", # chksum is (type, value)
|
checksum=chksum_value,
|
||||||
checksum_type=pkg.chksum[0] if pkg.chksum else "sha256",
|
checksum_type=chksum_type,
|
||||||
has_manpages=True if manpage_packages else False,
|
has_manpages=bool(manpage_packages),
|
||||||
)
|
)
|
||||||
|
|
||||||
packages.append(package)
|
packages.append(package)
|
||||||
seen_names.add(pkg_name)
|
seen_names.add(pkg_name)
|
||||||
|
|
||||||
logger.info(f"Found {len(packages)} packages to process")
|
logger.info(f"Found {len(packages)} packages to process")
|
||||||
return sorted(packages) # Sort by name for consistent ordering
|
return sorted(packages)
|
||||||
|
|
||||||
def download_package(self, package: Package) -> bool:
|
def download_package(self, package: Package) -> bool:
|
||||||
"""Download a single package.
|
"""Download a single package.
|
||||||
@@ -165,7 +178,6 @@ class RepoManager:
|
|||||||
download_path = self.download_dir / package.filename
|
download_path = self.download_dir / package.filename
|
||||||
package.download_path = download_path
|
package.download_path = download_path
|
||||||
|
|
||||||
# Skip if already downloaded
|
|
||||||
if download_path.exists():
|
if download_path.exists():
|
||||||
logger.debug(f"Package already downloaded: {package.filename}")
|
logger.debug(f"Package already downloaded: {package.filename}")
|
||||||
return True
|
return True
|
||||||
@@ -175,8 +187,7 @@ class RepoManager:
|
|||||||
response = requests.get(package.download_url, timeout=300, stream=True)
|
response = requests.get(package.download_url, timeout=300, stream=True)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
|
|
||||||
# Download with progress (optional: could add progress bar here)
|
with open(download_path, "wb") as f:
|
||||||
with open(download_path, 'wb') as f:
|
|
||||||
for chunk in response.iter_content(chunk_size=8192):
|
for chunk in response.iter_content(chunk_size=8192):
|
||||||
if chunk:
|
if chunk:
|
||||||
f.write(chunk)
|
f.write(chunk)
|
||||||
@@ -186,15 +197,12 @@ class RepoManager:
|
|||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error downloading {package.filename}: {e}")
|
logger.error(f"Error downloading {package.filename}: {e}")
|
||||||
# Clean up partial download
|
|
||||||
if download_path.exists():
|
if download_path.exists():
|
||||||
download_path.unlink()
|
download_path.unlink()
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def download_packages(
|
def download_packages(
|
||||||
self,
|
self, packages: List[Package], max_workers: int = 5
|
||||||
packages: List[Package],
|
|
||||||
max_workers: int = 5
|
|
||||||
) -> List[Package]:
|
) -> List[Package]:
|
||||||
"""Download multiple packages in parallel.
|
"""Download multiple packages in parallel.
|
||||||
|
|
||||||
@@ -208,13 +216,10 @@ class RepoManager:
|
|||||||
downloaded = []
|
downloaded = []
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||||
# Submit all download tasks
|
|
||||||
future_to_pkg = {
|
future_to_pkg = {
|
||||||
executor.submit(self.download_package, pkg): pkg
|
executor.submit(self.download_package, pkg): pkg for pkg in packages
|
||||||
for pkg in packages
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Process completed downloads
|
|
||||||
for future in as_completed(future_to_pkg):
|
for future in as_completed(future_to_pkg):
|
||||||
pkg = future_to_pkg[future]
|
pkg = future_to_pkg[future]
|
||||||
try:
|
try:
|
||||||
@@ -223,7 +228,9 @@ class RepoManager:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error processing {pkg.name}: {e}")
|
logger.error(f"Error processing {pkg.name}: {e}")
|
||||||
|
|
||||||
logger.info(f"Successfully downloaded {len(downloaded)}/{len(packages)} packages")
|
logger.info(
|
||||||
|
f"Successfully downloaded {len(downloaded)}/{len(packages)} packages"
|
||||||
|
)
|
||||||
return downloaded
|
return downloaded
|
||||||
|
|
||||||
def cleanup_package(self, package: Package):
|
def cleanup_package(self, package: Package):
|
||||||
|
|||||||
@@ -24,31 +24,26 @@ class Config:
|
|||||||
parallel_conversions: Number of parallel HTML conversions
|
parallel_conversions: Number of parallel HTML conversions
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# Repository configuration
|
|
||||||
base_url: str = "http://dl.rockylinux.org/"
|
base_url: str = "http://dl.rockylinux.org/"
|
||||||
content_dir: str = "pub/rocky"
|
content_dir: str = "pub/rocky"
|
||||||
versions: List[str] = None
|
versions: List[str] = None
|
||||||
architectures: List[str] = None
|
architectures: List[str] = None
|
||||||
repo_types: List[str] = None
|
repo_types: List[str] = None
|
||||||
|
|
||||||
# Directory configuration
|
|
||||||
download_dir: Path = Path("/data/tmp/downloads")
|
download_dir: Path = Path("/data/tmp/downloads")
|
||||||
extract_dir: Path = Path("/data/tmp/extracts")
|
extract_dir: Path = Path("/data/tmp/extracts")
|
||||||
output_dir: Path = Path("/data/html")
|
output_dir: Path = Path("/data/html")
|
||||||
|
|
||||||
# Cleanup options
|
|
||||||
keep_rpms: bool = False
|
keep_rpms: bool = False
|
||||||
keep_extracts: bool = False
|
keep_extracts: bool = False
|
||||||
|
|
||||||
# Performance options
|
|
||||||
parallel_downloads: int = 5
|
parallel_downloads: int = 5
|
||||||
parallel_conversions: int = 10
|
parallel_conversions: int = 10
|
||||||
|
|
||||||
# Filtering options
|
|
||||||
skip_sections: List[str] = None
|
skip_sections: List[str] = None
|
||||||
skip_packages: List[str] = None
|
skip_packages: List[str] = None
|
||||||
skip_languages: bool = True # Skip non-English languages by default
|
skip_languages: bool = True
|
||||||
allow_all_sections: bool = False # Override skip_sections if True
|
allow_all_sections: bool = False
|
||||||
|
|
||||||
def __post_init__(self):
|
def __post_init__(self):
|
||||||
"""Set defaults and ensure directories exist."""
|
"""Set defaults and ensure directories exist."""
|
||||||
@@ -56,20 +51,16 @@ class Config:
|
|||||||
self.versions = ["8.10", "9.6", "10.0"]
|
self.versions = ["8.10", "9.6", "10.0"]
|
||||||
|
|
||||||
if self.architectures is None:
|
if self.architectures is None:
|
||||||
# Man pages are arch-independent, so we just need one
|
|
||||||
# We prefer x86_64 as it's most common, fallback to others
|
|
||||||
self.architectures = ["x86_64", "aarch64", "ppc64le", "s390x"]
|
self.architectures = ["x86_64", "aarch64", "ppc64le", "s390x"]
|
||||||
|
|
||||||
if self.repo_types is None:
|
if self.repo_types is None:
|
||||||
self.repo_types = ["BaseOS", "AppStream"]
|
self.repo_types = ["BaseOS", "AppStream"]
|
||||||
|
|
||||||
# Set default skip sections (man3 library APIs)
|
|
||||||
if self.skip_sections is None and not self.allow_all_sections:
|
if self.skip_sections is None and not self.allow_all_sections:
|
||||||
self.skip_sections = ["3", "3p", "3pm"]
|
self.skip_sections = ["3", "3p", "3pm"]
|
||||||
elif self.allow_all_sections:
|
elif self.allow_all_sections:
|
||||||
self.skip_sections = []
|
self.skip_sections = []
|
||||||
|
|
||||||
# Set default skip packages (high-volume API docs)
|
|
||||||
if self.skip_packages is None:
|
if self.skip_packages is None:
|
||||||
self.skip_packages = [
|
self.skip_packages = [
|
||||||
"lapack",
|
"lapack",
|
||||||
@@ -77,7 +68,6 @@ class Config:
|
|||||||
"gl-manpages",
|
"gl-manpages",
|
||||||
]
|
]
|
||||||
|
|
||||||
# Ensure all paths are Path objects
|
|
||||||
self.download_dir = Path(self.download_dir)
|
self.download_dir = Path(self.download_dir)
|
||||||
self.extract_dir = Path(self.extract_dir)
|
self.extract_dir = Path(self.extract_dir)
|
||||||
self.output_dir = Path(self.output_dir)
|
self.output_dir = Path(self.output_dir)
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
import gzip
|
import gzip
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
from collections import defaultdict
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import List, Dict, Any
|
from typing import List, Dict, Any
|
||||||
|
|
||||||
@@ -33,10 +34,9 @@ class WebGenerator:
|
|||||||
self.output_dir = Path(output_dir)
|
self.output_dir = Path(output_dir)
|
||||||
self.output_dir.mkdir(parents=True, exist_ok=True)
|
self.output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
# Setup Jinja2 environment
|
|
||||||
self.env = Environment(
|
self.env = Environment(
|
||||||
loader=FileSystemLoader(str(self.template_dir)),
|
loader=FileSystemLoader(str(self.template_dir)),
|
||||||
autoescape=select_autoescape(['html', 'xml'])
|
autoescape=select_autoescape(["html", "xml"]),
|
||||||
)
|
)
|
||||||
|
|
||||||
def generate_manpage_html(self, man_file: ManFile, version: str) -> bool:
|
def generate_manpage_html(self, man_file: ManFile, version: str) -> bool:
|
||||||
@@ -54,7 +54,7 @@ class WebGenerator:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
try:
|
try:
|
||||||
template = self.env.get_template('manpage.html')
|
template = self.env.get_template("manpage.html")
|
||||||
|
|
||||||
html = template.render(
|
html = template.render(
|
||||||
title=f"{man_file.display_name} - {man_file.package_name} - Rocky Linux {version}",
|
title=f"{man_file.display_name} - {man_file.package_name} - Rocky Linux {version}",
|
||||||
@@ -62,17 +62,16 @@ class WebGenerator:
|
|||||||
package_name=man_file.package_name,
|
package_name=man_file.package_name,
|
||||||
version=version,
|
version=version,
|
||||||
section=man_file.section,
|
section=man_file.section,
|
||||||
language=man_file.language or 'en',
|
language=man_file.language or "en",
|
||||||
content=man_file.html_content
|
content=man_file.html_content,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Ensure output path is set
|
|
||||||
if not man_file.html_path:
|
if not man_file.html_path:
|
||||||
man_file.html_path = self._get_manpage_path(man_file, version)
|
man_file.html_path = self._get_manpage_path(man_file, version)
|
||||||
|
|
||||||
man_file.html_path.parent.mkdir(parents=True, exist_ok=True)
|
man_file.html_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
with open(man_file.html_path, 'w', encoding='utf-8') as f:
|
with open(man_file.html_path, "w", encoding="utf-8") as f:
|
||||||
f.write(html)
|
f.write(html)
|
||||||
|
|
||||||
return True
|
return True
|
||||||
@@ -92,19 +91,19 @@ class WebGenerator:
|
|||||||
True if successful
|
True if successful
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
template = self.env.get_template('index.html')
|
template = self.env.get_template("index.html")
|
||||||
|
|
||||||
html = template.render(
|
html = template.render(
|
||||||
title=f"Rocky Linux {version} Man Pages",
|
title=f"Rocky Linux {version} Man Pages",
|
||||||
version=version,
|
version=version,
|
||||||
total_pages=len(search_data),
|
total_pages=len(search_data),
|
||||||
packages=sorted(search_data.keys())
|
packages=sorted(search_data.keys()),
|
||||||
)
|
)
|
||||||
|
|
||||||
index_path = self.output_dir / version / 'index.html'
|
index_path = self.output_dir / version / "index.html"
|
||||||
index_path.parent.mkdir(parents=True, exist_ok=True)
|
index_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
with open(index_path, 'w', encoding='utf-8') as f:
|
with open(index_path, "w", encoding="utf-8") as f:
|
||||||
f.write(html)
|
f.write(html)
|
||||||
|
|
||||||
logger.info(f"Generated index for version {version}")
|
logger.info(f"Generated index for version {version}")
|
||||||
@@ -114,7 +113,9 @@ class WebGenerator:
|
|||||||
logger.error(f"Error generating index for {version}: {e}")
|
logger.error(f"Error generating index for {version}: {e}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def generate_packages_index(self, version: str, search_data: Dict[str, Any]) -> bool:
|
def generate_packages_index(
|
||||||
|
self, version: str, search_data: Dict[str, Any]
|
||||||
|
) -> bool:
|
||||||
"""Generate full packages index page.
|
"""Generate full packages index page.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -125,39 +126,32 @@ class WebGenerator:
|
|||||||
True if successful
|
True if successful
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
# Group packages by first letter
|
packages_by_letter = defaultdict(list)
|
||||||
packages_by_letter = {}
|
|
||||||
|
|
||||||
for pkg_name, pages in search_data.items():
|
for pkg_name, pages in search_data.items():
|
||||||
first_char = pkg_name[0].upper()
|
first_char = pkg_name[0].upper()
|
||||||
if not first_char.isalpha():
|
if not first_char.isalpha():
|
||||||
first_char = 'other'
|
first_char = "other"
|
||||||
|
packages_by_letter[first_char].append(
|
||||||
|
{"name": pkg_name, "count": len(pages)}
|
||||||
|
)
|
||||||
|
|
||||||
if first_char not in packages_by_letter:
|
for packages in packages_by_letter.values():
|
||||||
packages_by_letter[first_char] = []
|
packages.sort(key=lambda x: x["name"])
|
||||||
|
|
||||||
packages_by_letter[first_char].append({
|
template = self.env.get_template("packages.html")
|
||||||
'name': pkg_name,
|
|
||||||
'count': len(pages)
|
|
||||||
})
|
|
||||||
|
|
||||||
# Sort packages within each letter
|
|
||||||
for letter in packages_by_letter:
|
|
||||||
packages_by_letter[letter].sort(key=lambda x: x['name'])
|
|
||||||
|
|
||||||
template = self.env.get_template('packages.html')
|
|
||||||
|
|
||||||
html = template.render(
|
html = template.render(
|
||||||
title=f"All Packages - Rocky Linux {version}",
|
title=f"All Packages - Rocky Linux {version}",
|
||||||
version=version,
|
version=version,
|
||||||
total_packages=len(search_data),
|
total_packages=len(search_data),
|
||||||
packages_by_letter=packages_by_letter
|
packages_by_letter=packages_by_letter,
|
||||||
)
|
)
|
||||||
|
|
||||||
output_path = self.output_dir / version / 'packages.html'
|
output_path = self.output_dir / version / "packages.html"
|
||||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
with open(output_path, 'w', encoding='utf-8') as f:
|
with open(output_path, "w", encoding="utf-8") as f:
|
||||||
f.write(html)
|
f.write(html)
|
||||||
|
|
||||||
logger.info(f"Generated packages index for version {version}")
|
logger.info(f"Generated packages index for version {version}")
|
||||||
@@ -168,9 +162,7 @@ class WebGenerator:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
def generate_search_index(
|
def generate_search_index(
|
||||||
self,
|
self, man_files: List[ManFile], version: str
|
||||||
man_files: List[ManFile],
|
|
||||||
version: str
|
|
||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
"""Generate search index from man files.
|
"""Generate search index from man files.
|
||||||
|
|
||||||
@@ -189,17 +181,15 @@ class WebGenerator:
|
|||||||
if pkg_name not in index:
|
if pkg_name not in index:
|
||||||
index[pkg_name] = {}
|
index[pkg_name] = {}
|
||||||
|
|
||||||
# Create entry for this man page
|
|
||||||
entry = {
|
entry = {
|
||||||
'name': man_file.name,
|
"name": man_file.name,
|
||||||
'section': man_file.section,
|
"section": man_file.section,
|
||||||
'display_name': man_file.display_name,
|
"display_name": man_file.display_name,
|
||||||
'language': man_file.language or 'en',
|
"language": man_file.language or "en",
|
||||||
'url': man_file.uri_path,
|
"url": man_file.uri_path,
|
||||||
'full_name': f"{man_file.package_name} - {man_file.display_name}"
|
"full_name": f"{man_file.package_name} - {man_file.display_name}",
|
||||||
}
|
}
|
||||||
|
|
||||||
# Use display name as key (handles duplicates with different sections)
|
|
||||||
key = man_file.display_name
|
key = man_file.display_name
|
||||||
if man_file.language:
|
if man_file.language:
|
||||||
key = f"{key}.{man_file.language}"
|
key = f"{key}.{man_file.language}"
|
||||||
@@ -222,18 +212,14 @@ class WebGenerator:
|
|||||||
version_dir = self.output_dir / version
|
version_dir = self.output_dir / version
|
||||||
version_dir.mkdir(parents=True, exist_ok=True)
|
version_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
json_path = version_dir / 'search.json'
|
json_path = version_dir / "search.json"
|
||||||
gz_path = version_dir / 'search.json.gz'
|
gz_path = version_dir / "search.json.gz"
|
||||||
|
|
||||||
# Sort for consistency
|
|
||||||
sorted_index = {k: index[k] for k in sorted(index)}
|
sorted_index = {k: index[k] for k in sorted(index)}
|
||||||
|
|
||||||
# Save plain JSON
|
with open(json_path, "w", encoding="utf-8") as f:
|
||||||
with open(json_path, 'w', encoding='utf-8') as f:
|
|
||||||
json.dump(sorted_index, f, indent=2)
|
json.dump(sorted_index, f, indent=2)
|
||||||
|
|
||||||
# Save gzipped JSON
|
with gzip.open(gz_path, "wt", encoding="utf-8") as f:
|
||||||
with gzip.open(gz_path, 'wt', encoding='utf-8') as f:
|
|
||||||
json.dump(sorted_index, f)
|
json.dump(sorted_index, f)
|
||||||
|
|
||||||
logger.info(f"Saved search index for {version} ({len(index)} packages)")
|
logger.info(f"Saved search index for {version} ({len(index)} packages)")
|
||||||
@@ -269,24 +255,38 @@ class WebGenerator:
|
|||||||
True if successful
|
True if successful
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
template = self.env.get_template('root.html')
|
template = self.env.get_template("root.html")
|
||||||
|
|
||||||
# Sort versions numerically (e.g., 8.10, 9.6, 10.0)
|
major_to_minors = defaultdict(list)
|
||||||
def version_key(v):
|
for v in versions:
|
||||||
try:
|
try:
|
||||||
parts = v.split('.')
|
major, minor = v.split(".")
|
||||||
return tuple(int(p) for p in parts)
|
major_to_minors[major].append(minor)
|
||||||
except (ValueError, AttributeError):
|
except ValueError:
|
||||||
return (0, 0)
|
continue
|
||||||
|
|
||||||
|
sorted_majors = sorted(major_to_minors, key=int)
|
||||||
|
max_minors = max((len(major_to_minors[m]) for m in sorted_majors), default=0)
|
||||||
|
num_columns = len(sorted_majors)
|
||||||
|
|
||||||
|
version_rows = []
|
||||||
|
for minor_idx in range(max_minors):
|
||||||
|
row = []
|
||||||
|
for major in sorted_majors:
|
||||||
|
minors_list = sorted(major_to_minors[major], key=int, reverse=True)
|
||||||
|
if minor_idx < len(minors_list):
|
||||||
|
row.append((major, minors_list[minor_idx]))
|
||||||
|
else:
|
||||||
|
row.append(None)
|
||||||
|
version_rows.append(row)
|
||||||
|
|
||||||
html = template.render(
|
html = template.render(
|
||||||
title="Rocky Linux Man Pages",
|
title="Rocky Linux Man Pages", version_rows=version_rows, num_columns=num_columns
|
||||||
versions=sorted(versions, key=version_key)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
index_path = self.output_dir / 'index.html'
|
index_path = self.output_dir / "index.html"
|
||||||
|
|
||||||
with open(index_path, 'w', encoding='utf-8') as f:
|
with open(index_path, "w", encoding="utf-8") as f:
|
||||||
f.write(html)
|
f.write(html)
|
||||||
|
|
||||||
logger.info("Generated root index page")
|
logger.info("Generated root index page")
|
||||||
@@ -295,3 +295,28 @@ class WebGenerator:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error generating root index: {e}")
|
logger.error(f"Error generating root index: {e}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
def generate_404_page(self) -> bool:
|
||||||
|
"""Generate 404 error page.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if successful
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
template = self.env.get_template("404.html")
|
||||||
|
|
||||||
|
html = template.render(
|
||||||
|
title="404 - Page Not Found"
|
||||||
|
)
|
||||||
|
|
||||||
|
error_path = self.output_dir / "404.html"
|
||||||
|
|
||||||
|
with open(error_path, "w", encoding="utf-8") as f:
|
||||||
|
f.write(html)
|
||||||
|
|
||||||
|
logger.info("Generated 404 page")
|
||||||
|
return True
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error generating 404 page: {e}")
|
||||||
|
return False
|
||||||
|
|||||||
@@ -0,0 +1,137 @@
|
|||||||
|
{% extends "base.html" %}
|
||||||
|
|
||||||
|
{% block header_title %}Rocky Linux Man Pages{% endblock %}
|
||||||
|
{% block header_subtitle %}Man page documentation for Rocky Linux packages{% endblock %}
|
||||||
|
|
||||||
|
{% block extra_css %}
|
||||||
|
.error-container {
|
||||||
|
text-align: center;
|
||||||
|
padding: 4rem 2rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.error-code {
|
||||||
|
font-size: 8rem;
|
||||||
|
font-weight: 700;
|
||||||
|
color: var(--accent-primary);
|
||||||
|
line-height: 1;
|
||||||
|
margin-bottom: 1rem;
|
||||||
|
font-family: "JetBrains Mono", monospace;
|
||||||
|
}
|
||||||
|
|
||||||
|
.error-message {
|
||||||
|
font-size: 1.5rem;
|
||||||
|
color: var(--text-primary);
|
||||||
|
margin-bottom: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.error-description {
|
||||||
|
color: var(--text-secondary);
|
||||||
|
margin-bottom: 2rem;
|
||||||
|
max-width: 600px;
|
||||||
|
margin-left: auto;
|
||||||
|
margin-right: auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.suggestions {
|
||||||
|
max-width: 600px;
|
||||||
|
margin: 2rem auto;
|
||||||
|
text-align: left;
|
||||||
|
}
|
||||||
|
|
||||||
|
.suggestions h3 {
|
||||||
|
color: var(--text-primary);
|
||||||
|
margin-bottom: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.suggestions ul {
|
||||||
|
list-style: none;
|
||||||
|
padding: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.suggestions li {
|
||||||
|
margin-bottom: 0.75rem;
|
||||||
|
padding-left: 1.5rem;
|
||||||
|
position: relative;
|
||||||
|
}
|
||||||
|
|
||||||
|
.suggestions li::before {
|
||||||
|
content: "→";
|
||||||
|
position: absolute;
|
||||||
|
left: 0;
|
||||||
|
color: var(--accent-primary);
|
||||||
|
}
|
||||||
|
|
||||||
|
.back-button {
|
||||||
|
display: inline-block;
|
||||||
|
padding: 0.75rem 1.5rem;
|
||||||
|
background: var(--accent-primary);
|
||||||
|
color: white;
|
||||||
|
text-decoration: none;
|
||||||
|
border-radius: 6px;
|
||||||
|
font-weight: 500;
|
||||||
|
transition: all 0.2s;
|
||||||
|
margin-top: 2rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.back-button:hover {
|
||||||
|
background: var(--accent-secondary);
|
||||||
|
transform: translateY(-2px);
|
||||||
|
text-decoration: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (max-width: 768px) {
|
||||||
|
.error-code {
|
||||||
|
font-size: 5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.error-message {
|
||||||
|
font-size: 1.25rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.error-container {
|
||||||
|
padding: 3rem 1rem;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (max-width: 480px) {
|
||||||
|
.error-code {
|
||||||
|
font-size: 4rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.error-message {
|
||||||
|
font-size: 1.1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.error-container {
|
||||||
|
padding: 2rem 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.suggestions {
|
||||||
|
padding: 0 1rem;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
{% endblock %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
<div class="content">
|
||||||
|
<div class="error-container">
|
||||||
|
<div class="error-code">404</div>
|
||||||
|
<div class="error-message">Page Not Found</div>
|
||||||
|
<div class="error-description">
|
||||||
|
The page you're looking for doesn't exist or may have been moved.
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="suggestions">
|
||||||
|
<h3>Suggestions:</h3>
|
||||||
|
<ul>
|
||||||
|
<li>Check the URL for typos</li>
|
||||||
|
<li>Return to the <a href="/">home page</a> and navigate from there</li>
|
||||||
|
<li>Use the search feature on the version index page</li>
|
||||||
|
<li>The man page may be in a different version of Rocky Linux</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<a href="/" class="back-button">Go to Home Page</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endblock %}
|
||||||
+1
-4
@@ -255,10 +255,7 @@
|
|||||||
Search by <a href="https://fusejs.io/" target="_blank">Fuse.js</a>
|
Search by <a href="https://fusejs.io/" target="_blank">Fuse.js</a>
|
||||||
</p>
|
</p>
|
||||||
<p style="margin-top: 0.5rem; font-size: 0.85rem;">
|
<p style="margin-top: 0.5rem; font-size: 0.85rem;">
|
||||||
Rocky Linux™ is a trademark of the Rocky Enterprise Software Foundation.
|
Rocky Linux is a trademark of the Rocky Enterprise Software Foundation.
|
||||||
</p>
|
|
||||||
<p style="margin-top: 0.5rem; font-size: 0.85rem;">
|
|
||||||
This tool is open source (MIT License). See THIRD-PARTY-LICENSES.md for attributions.
|
|
||||||
</p>
|
</p>
|
||||||
</footer>
|
</footer>
|
||||||
|
|
||||||
|
|||||||
@@ -112,6 +112,47 @@ font-size: 0.9em;
|
|||||||
color: var(--success);
|
color: var(--success);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* OPTIONS section specific styling */
|
||||||
|
/* Style paragraphs that contain option flags (b tags followed by i tags or immediately followed by Bd-indent) */
|
||||||
|
.man-content section.Sh p.Pp:has(+ .Bd-indent) {
|
||||||
|
font-weight: 600;
|
||||||
|
font-size: 1.05em;
|
||||||
|
margin-top: 1.5rem;
|
||||||
|
margin-bottom: 0.5rem;
|
||||||
|
padding: 0.5rem 0.75rem;
|
||||||
|
background: linear-gradient(90deg, var(--bg-tertiary) 0%, transparent 100%);
|
||||||
|
border-left: 3px solid var(--accent-primary);
|
||||||
|
}
|
||||||
|
|
||||||
|
.man-content section.Sh p.Pp:has(+ .Bd-indent) b {
|
||||||
|
color: var(--accent-primary);
|
||||||
|
font-size: 1em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.man-content section.Sh p.Pp:has(+ .Bd-indent) i {
|
||||||
|
color: var(--text-secondary);
|
||||||
|
font-style: italic;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Indented description blocks */
|
||||||
|
.man-content .Bd-indent {
|
||||||
|
margin-left: 2.5rem;
|
||||||
|
margin-bottom: 1.5rem;
|
||||||
|
padding-left: 1rem;
|
||||||
|
border-left: 2px solid var(--border-color);
|
||||||
|
color: var(--text-primary);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Add spacing between nested paragraphs in descriptions */
|
||||||
|
.man-content .Bd-indent > p.Pp {
|
||||||
|
margin-top: 0.75rem;
|
||||||
|
margin-bottom: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.man-content .Bd-indent > p.Pp:first-child {
|
||||||
|
margin-top: 0;
|
||||||
|
}
|
||||||
|
|
||||||
.man-content pre {
|
.man-content pre {
|
||||||
background-color: var(--bg-primary);
|
background-color: var(--bg-primary);
|
||||||
border: 1px solid var(--border-color);
|
border: 1px solid var(--border-color);
|
||||||
@@ -215,6 +256,16 @@ margin-left: 1rem;
|
|||||||
.man-content .Bl-dash {
|
.man-content .Bl-dash {
|
||||||
padding-left: 1rem;
|
padding-left: 1rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.man-content section.Sh p.Pp:has(+ .Bd-indent) {
|
||||||
|
font-size: 1em;
|
||||||
|
padding: 0.4rem 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.man-content .Bd-indent {
|
||||||
|
margin-left: 1.5rem;
|
||||||
|
padding-left: 0.75rem;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@media (max-width: 480px) {
|
@media (max-width: 480px) {
|
||||||
|
|||||||
+46
-9
@@ -1,7 +1,7 @@
|
|||||||
{% extends "base.html" %}
|
{% extends "base.html" %}
|
||||||
|
|
||||||
{% block header_title %}Rocky Linux™ Man Pages{% endblock %}
|
{% block header_title %}Rocky Linux Man Pages{% endblock %}
|
||||||
{% block header_subtitle %}Man page documentation for Rocky Linux™ packages{% endblock %}
|
{% block header_subtitle %}Man page documentation for Rocky Linux packages{% endblock %}
|
||||||
|
|
||||||
{% block extra_css %}
|
{% block extra_css %}
|
||||||
.logo-container {
|
.logo-container {
|
||||||
@@ -15,9 +15,11 @@
|
|||||||
height: auto;
|
height: auto;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
.version-grid {
|
.version-grid {
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
|
grid-template-columns: repeat({{ num_columns }}, 1fr);
|
||||||
gap: 1.5rem;
|
gap: 1.5rem;
|
||||||
margin-top: 2rem;
|
margin-top: 2rem;
|
||||||
}
|
}
|
||||||
@@ -32,7 +34,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
.version-grid {
|
.version-grid {
|
||||||
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
grid-template-columns: 1fr;
|
||||||
gap: 1rem;
|
gap: 1rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -40,9 +42,21 @@
|
|||||||
padding: 1.5rem;
|
padding: 1.5rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.version-card.small {
|
||||||
|
padding: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.version-card.small {
|
||||||
|
padding: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
.version-number {
|
.version-number {
|
||||||
font-size: 2rem;
|
font-size: 2rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.version-card.small .version-number {
|
||||||
|
font-size: 1.5rem;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@media (max-width: 480px) {
|
@media (max-width: 480px) {
|
||||||
@@ -55,6 +69,10 @@
|
|||||||
gap: 1rem;
|
gap: 1rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.version-card.small {
|
||||||
|
padding: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
.intro {
|
.intro {
|
||||||
font-size: 0.9rem;
|
font-size: 0.9rem;
|
||||||
}
|
}
|
||||||
@@ -71,6 +89,15 @@
|
|||||||
display: block;
|
display: block;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.version-card.small {
|
||||||
|
padding: 1rem;
|
||||||
|
opacity: 0.7;
|
||||||
|
}
|
||||||
|
|
||||||
|
.version-card.small .version-number {
|
||||||
|
font-size: 1.8rem;
|
||||||
|
}
|
||||||
|
|
||||||
.version-card:hover {
|
.version-card:hover {
|
||||||
transform: translateY(-2px);
|
transform: translateY(-2px);
|
||||||
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.3);
|
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.3);
|
||||||
@@ -128,19 +155,29 @@
|
|||||||
|
|
||||||
<div class="intro">
|
<div class="intro">
|
||||||
<p>
|
<p>
|
||||||
Man page documentation for packages in the Rocky Linux™ BaseOS and AppStream repositories.
|
Man page documentation for packages in the Rocky Linux BaseOS and AppStream repositories.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="version-section">
|
<div class="version-section">
|
||||||
<h2>Select Version</h2>
|
<h2>Select Version</h2>
|
||||||
<div class="version-grid">
|
<div class="version-grid">
|
||||||
{% for version in versions %}
|
{% for row in version_rows %}
|
||||||
<a href="{{ version }}/index.html" class="version-card">
|
{% set outer_loop = loop %}
|
||||||
<div class="version-number">{{ version }}</div>
|
{% for item in row %}
|
||||||
<div class="version-label">Rocky Linux™</div>
|
{% if item %}
|
||||||
|
{% set major, minor = item %}
|
||||||
|
<a href="{{ major }}.{{ minor }}/index.html" class="version-card{% if not outer_loop.first %} small{% endif %}">
|
||||||
|
<div class="version-number">{{ major }}.{{ minor }}</div>
|
||||||
|
{% if outer_loop.first %}
|
||||||
|
<div class="version-label">Rocky Linux</div>
|
||||||
<div class="version-browse">Browse man pages →</div>
|
<div class="version-browse">Browse man pages →</div>
|
||||||
|
{% endif %}
|
||||||
</a>
|
</a>
|
||||||
|
{% else %}
|
||||||
|
<div></div>
|
||||||
|
{% endif %}
|
||||||
|
{% endfor %}
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
Reference in New Issue
Block a user