fedora-metalink-proxy/app.py

import os
import requests
import time
from collections import defaultdict
from flask import Flask, render_template, request, redirect, url_for, Response
from dotenv import load_dotenv
from bs4 import BeautifulSoup
import pandas as pd
import matplotlib.pyplot as plt
from io import BytesIO
import base64

# Load environment variables from .env file
load_dotenv()

app = Flask(__name__, template_folder='templates')

# Data structure to store request statistics
request_stats = defaultdict(lambda: defaultdict(int))

def get_config():
    """Retrieve configuration from environment variables."""
    return {
        "excluded_countries": os.getenv('EXCLUDED_COUNTRIES', '[]'),
        "preferred_protocols": os.getenv('PREFERRED_PROTOCOLS', '["https", "http"]'),
        "preferred_types": os.getenv('PREFERRED_TYPES', '["https", "http"]'),
        "min_preference": os.getenv('MIN_PREFERENCE', '0')
    }

def log_request(repo, arch, protocol):
    """Log request statistics."""
    timestamp = int(time.time())
    hour = time.strftime('%Y-%m-%d %H', time.localtime(timestamp))
    day = time.strftime('%Y-%m-%d', time.localtime(timestamp))
    week = time.strftime('%Y-%U', time.localtime(timestamp))
    month = time.strftime('%Y-%m', time.localtime(timestamp))

    request_stats[hour][(repo, arch, protocol)] += 1
    request_stats[day][(repo, arch, protocol)] += 1
    request_stats[week][(repo, arch, protocol)] += 1
    request_stats[month][(repo, arch, protocol)] += 1
@app.route('/metalink')
def get_metalink():
    # Get query parameters from the request
    repo = request.args.get('repo')
    arch = request.args.get('arch')

    # Check if required parameters are provided
    if not repo or not arch:
        return "Error: Missing 'repo' or 'arch' parameter", 400

    # Construct the metalink URL using the provided parameters
    metalink_url = f'https://mirrors.fedoraproject.org/metalink?repo={repo}&arch={arch}'

    # Fetch the metalink file from the constructed URL
    response = requests.get(metalink_url)
    metalink_content = response.content

    # Get the filtering criteria from environment variables
    config = get_config()
    excluded_countries = eval(config['excluded_countries'])
    preferred_protocols = eval(config['preferred_protocols'])
    preferred_types = eval(config['preferred_types'])
    min_preference = int(config['min_preference'])

    # Filter out the URLs based on the criteria
    filtered_content = filter_urls(metalink_content, excluded_countries, preferred_protocols, preferred_types, min_preference)

    # Return the filtered content as a response
    return Response(filtered_content, mimetype='application/xml')

def filter_urls(content, excluded_countries, preferred_protocols, preferred_types, min_preference):
    # Parse the XML content
    soup = BeautifulSoup(content, 'xml')

    # Find all URL elements
    urls = soup.find_all('url')

    # Iterate over URLs and remove those that do not meet the criteria
    for url in urls:
        location = url.get('location')
        protocol = url.get('protocol')
        type_ = url.get('type')
        preference = int(url.get('preference', 0))

        if (location in excluded_countries or
            protocol not in preferred_protocols or
            type_ not in preferred_types or
            preference < min_preference):
            url.decompose()

    # Convert the BeautifulSoup object back to a string and clean up
    filtered_content = str(soup)
    filtered_content = '\n'.join(line for line in filtered_content.splitlines() if line.strip())

    return filtered_content

@app.route('/dash')
@app.route('/dash/stats')
def stats():
    # Generate some statistics
    stats_data = {
        'hourly': request_stats.get(time.strftime('%Y-%m-%d %H', time.localtime()), {}),
        'daily': request_stats.get(time.strftime('%Y-%m-%d', time.localtime()), {}),
        'weekly': request_stats.get(time.strftime('%Y-%U', time.localtime()), {}),
        'monthly': request_stats.get(time.strftime('%Y-%m', time.localtime()), {})
    }

    # Convert stats data to a DataFrame for easier manipulation
    df = pd.DataFrame.from_dict(stats_data, orient='index').fillna(0)

    # Generate a simple plot
    plt.figure(figsize=(10, 6))
    df.sum(axis=1).plot(kind='bar')
    plt.title('Request Statistics')
    plt.ylabel('Number of Requests')

    # Save plot to a BytesIO object
    img = BytesIO()
    plt.savefig(img, format='png')
    plt.close()
    img.seek(0)

    # Encode the plot to base64 for embedding in HTML
    plot_url = base64.b64encode(img.getvalue()).decode('utf8')

    return render_template('stats.html', plot_url=plot_url)

@app.route('/dash/config', methods=['GET', 'POST'])
def config():
    if request.method == 'POST':
        # Update environment variables with form data
        os.environ['EXCLUDED_COUNTRIES'] = request.form.get('excluded_countries', '')
        os.environ['PREFERRED_PROTOCOLS'] = request.form.get('preferred_protocols', '')
        os.environ['PREFERRED_TYPES'] = request.form.get('preferred_types', '')
        os.environ['MIN_PREFERENCE'] = request.form.get('min_preference', '')

        return redirect(url_for('config'))

    config = get_config()
    return render_template('config.html', **config)

@app.route('/dash/logs')
def logs():
    return render_template('logs.html')

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8182)