import os import requests import time from collections import defaultdict from flask import Flask, render_template, request, redirect, url_for, Response from dotenv import load_dotenv from bs4 import BeautifulSoup import pandas as pd import matplotlib.pyplot as plt from io import BytesIO import base64 # Load environment variables from .env file load_dotenv() app = Flask(__name__, template_folder='templates') # Data structure to store request statistics request_stats = defaultdict(lambda: defaultdict(int)) def get_config(): """Retrieve configuration from environment variables.""" return { "excluded_countries": os.getenv('EXCLUDED_COUNTRIES', '[]'), "preferred_protocols": os.getenv('PREFERRED_PROTOCOLS', '["https", "http"]'), "preferred_types": os.getenv('PREFERRED_TYPES', '["https", "http"]'), "min_preference": os.getenv('MIN_PREFERENCE', '0') } def log_request(repo, arch, protocol): """Log request statistics.""" timestamp = int(time.time()) hour = time.strftime('%Y-%m-%d %H', time.localtime(timestamp)) day = time.strftime('%Y-%m-%d', time.localtime(timestamp)) week = time.strftime('%Y-%U', time.localtime(timestamp)) month = time.strftime('%Y-%m', time.localtime(timestamp)) request_stats[hour][(repo, arch, protocol)] += 1 request_stats[day][(repo, arch, protocol)] += 1 request_stats[week][(repo, arch, protocol)] += 1 request_stats[month][(repo, arch, protocol)] += 1 @app.route('/metalink') def get_metalink(): # Get query parameters from the request repo = request.args.get('repo') arch = request.args.get('arch') # Check if required parameters are provided if not repo or not arch: return "Error: Missing 'repo' or 'arch' parameter", 400 # Construct the metalink URL using the provided parameters metalink_url = f'https://mirrors.fedoraproject.org/metalink?repo={repo}&arch={arch}' # Fetch the metalink file from the constructed URL response = requests.get(metalink_url) metalink_content = response.content # Get the filtering criteria from environment variables config = get_config() excluded_countries = eval(config['excluded_countries']) preferred_protocols = eval(config['preferred_protocols']) preferred_types = eval(config['preferred_types']) min_preference = int(config['min_preference']) # Filter out the URLs based on the criteria filtered_content = filter_urls(metalink_content, excluded_countries, preferred_protocols, preferred_types, min_preference) # Return the filtered content as a response return Response(filtered_content, mimetype='application/xml') def filter_urls(content, excluded_countries, preferred_protocols, preferred_types, min_preference): # Parse the XML content soup = BeautifulSoup(content, 'xml') # Find all URL elements urls = soup.find_all('url') # Iterate over URLs and remove those that do not meet the criteria for url in urls: location = url.get('location') protocol = url.get('protocol') type_ = url.get('type') preference = int(url.get('preference', 0)) if (location in excluded_countries or protocol not in preferred_protocols or type_ not in preferred_types or preference < min_preference): url.decompose() # Convert the BeautifulSoup object back to a string and clean up filtered_content = str(soup) filtered_content = '\n'.join(line for line in filtered_content.splitlines() if line.strip()) return filtered_content @app.route('/dash') @app.route('/dash/stats') def stats(): # Generate some statistics stats_data = { 'hourly': request_stats.get(time.strftime('%Y-%m-%d %H', time.localtime()), {}), 'daily': request_stats.get(time.strftime('%Y-%m-%d', time.localtime()), {}), 'weekly': request_stats.get(time.strftime('%Y-%U', time.localtime()), {}), 'monthly': request_stats.get(time.strftime('%Y-%m', time.localtime()), {}) } # Convert stats data to a DataFrame for easier manipulation df = pd.DataFrame.from_dict(stats_data, orient='index').fillna(0) # Generate a simple plot plt.figure(figsize=(10, 6)) df.sum(axis=1).plot(kind='bar') plt.title('Request Statistics') plt.ylabel('Number of Requests') # Save plot to a BytesIO object img = BytesIO() plt.savefig(img, format='png') plt.close() img.seek(0) # Encode the plot to base64 for embedding in HTML plot_url = base64.b64encode(img.getvalue()).decode('utf8') return render_template('stats.html', plot_url=plot_url) @app.route('/dash/config', methods=['GET', 'POST']) def config(): if request.method == 'POST': # Update environment variables with form data os.environ['EXCLUDED_COUNTRIES'] = request.form.get('excluded_countries', '') os.environ['PREFERRED_PROTOCOLS'] = request.form.get('preferred_protocols', '') os.environ['PREFERRED_TYPES'] = request.form.get('preferred_types', '') os.environ['MIN_PREFERENCE'] = request.form.get('min_preference', '') return redirect(url_for('config')) config = get_config() return render_template('config.html', **config) @app.route('/dash/logs') def logs(): return render_template('logs.html') if __name__ == '__main__': app.run(host='0.0.0.0', port=8182)