Backup of releases and the addition of search/sorting in the repositories tab

README.md

@@ -11,6 +11,11 @@
 <details>
   <summary>Open for sort of CHANGELOGS?</summary>
 
+**V1.03 - Releases and sorting**  
+Added the option to backup releases (meaning all files in an release). Options are None, latest release, last 5 releases or all releases
+Added simplistic search in the repositories tab
+Added sorting in repositories tab
+
 **V1.02 - cap-drop=all**  
 Added the option to make use of --cap-drop=ALL as long as you specify an user in an parameter ( --user xx:xx)
 

app.py

@@ -10,7 +10,7 @@
 from apscheduler.triggers.cron import CronTrigger
 import logging
 from backup_service import BackupService
-from models import db, User, Repository, BackupJob, PasswordResetCode
+from models import db, User, Repository, BackupJob, PasswordResetCode, RELEASE_MODES
 import atexit
 
 # Configure logging
@@ -316,6 +316,11 @@ def ensure_scheduler_initialized():
             _scheduler_initialized = True
             logger.info("Scheduler initialization completed")
 
+def get_release_mode_from_form():
+    """Read the release backup mode from the submitted form, defaulting to no releases"""
+    release_mode = request.form.get('release_mode', 'none')
+    return release_mode if release_mode in RELEASE_MODES else 'none'
+
 @login_manager.user_loader
 def load_user(user_id):
     return User.query.get(int(user_id))
@@ -445,8 +450,8 @@ def reset_password():
 @app.route('/repositories')
 @login_required
 def repositories():
-    repos = Repository.query.filter_by(user_id=current_user.id).all()
-    
+    repos = Repository.query.filter_by(user_id=current_user.id).order_by(Repository.id).all()
+
     # Get backup job status
     today_start = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
     running_jobs = BackupJob.query.filter_by(user_id=current_user.id, status='running').all()
@@ -538,6 +543,7 @@ def add_repository():
             backup_format=backup_format,
             schedule_type=schedule_type,
             retention_count=retention_count,
+            release_mode=get_release_mode_from_form(),
             custom_interval=custom_interval,
             custom_unit=custom_unit,
             custom_hour=custom_hour,
@@ -566,7 +572,8 @@ def add_repositories_by_username():
         backup_format = request.form.get('backup_format', 'folder')
         schedule_type = request.form.get('schedule_type', 'daily')
         retention_count = int(request.form.get('retention_count', 5))
-
+        release_mode = get_release_mode_from_form()
+
         if not github_username:
             flash('Please provide a GitHub username', 'error')
             return render_template('add_by_username.html')
@@ -637,6 +644,7 @@ def add_repositories_by_username():
                         backup_format=backup_format,
                         schedule_type=schedule_type,
                         retention_count=retention_count,
+                        release_mode=release_mode,
                         is_active=True
                     )
 
@@ -714,6 +722,7 @@ def edit_repository(repo_id):
         repository.backup_format = request.form['backup_format']
         repository.schedule_type = request.form['schedule_type']
         repository.retention_count = int(request.form['retention_count'])
+        repository.release_mode = get_release_mode_from_form()
         repository.is_active = 'is_active' in request.form
 
         # Handle custom schedule fields

backup_service.py

@@ -1,18 +1,23 @@
 import os
+import re
 import git
 import shutil
+import requests
 import zipfile
 import tarfile
 import logging
 from datetime import datetime, timedelta
 from pathlib import Path
 from github import Github
-from models import db, BackupJob
+from models import db, BackupJob, RELEASE_MODES
 from urllib.parse import urlparse
 
 logger = logging.getLogger(__name__)
 
 class BackupService:
+    # Folder inside each backup that holds downloaded release files
+    RELEASES_DIR_NAME = '_releases'
+
     def __init__(self):
         self.backup_base_dir = Path('/app/backups')
         self.backup_base_dir.mkdir(exist_ok=True)
@@ -153,7 +158,10 @@ def backup_repository(self, repository):
             temp_clone_dir.mkdir(parents=True, exist_ok=False)
 
             self._clone_repository(repository, temp_clone_dir)
-
+
+            # Download release files into the backup if enabled for this repository
+            self._download_releases(repository, temp_clone_dir / self.RELEASES_DIR_NAME)
+
             # Create backup in specified format
             backup_path = self._create_backup(
                 temp_clone_dir, 
@@ -290,6 +298,100 @@ def _clone_repository(self, repository, clone_dir):
                     pass
             raise e
 
+    def _parse_owner_repo(self, repo_url):
+        """Return (owner, repo_name) for a GitHub URL, or None if it is not a GitHub URL"""
+        if repo_url.startswith('git@'):
+            # git@github.com:owner/repo.git
+            host, _, path = repo_url[len('git@'):].partition(':')
+        else:
+            parsed = urlparse(repo_url)
+            host, path = parsed.hostname or '', parsed.path
+
+        if host.lower() != 'github.com':
+            return None
+
+        path_parts = path.strip('/').split('/')
+        if len(path_parts) < 2:
+            return None
+
+        repo_name = path_parts[1]
+        if repo_name.endswith('.git'):
+            repo_name = repo_name[:-4]
+        return path_parts[0], repo_name
+
+    def _safe_filename(self, name):
+        """Make a release tag or asset name safe to use as a single path component"""
+        safe = re.sub(r'[^A-Za-z0-9._-]', '_', name).strip('.')
+        return safe or 'unnamed'
+
+    def _download_file(self, url, destination, headers):
+        """Stream a file from the GitHub API to disk"""
+        with requests.get(url, headers=headers, stream=True, timeout=(30, 300), allow_redirects=True) as response:
+            response.raise_for_status()
+            with open(destination, 'wb') as f:
+                for chunk in response.iter_content(chunk_size=1024 * 1024):
+                    if chunk:
+                        f.write(chunk)
+
+    def _download_releases(self, repository, releases_dir):
+        """Download release assets and source archives according to the repository's release mode"""
+        release_mode = repository.release_mode or 'none'
+        if release_mode not in RELEASE_MODES:
+            logger.warning(f"Unknown release mode '{release_mode}' for {repository.name}, skipping releases")
+            return
+        if release_mode == 'none':
+            return
+
+        owner_repo = self._parse_owner_repo(repository.url)
+        if not owner_repo:
+            raise Exception(f"Cannot back up releases: not a GitHub repository URL: {repository.url}")
+        owner, repo_name = owner_repo
+
+        token = repository.github_token.strip() if repository.github_token else ''
+        g = Github(token) if token else Github()
+        gh_repo = g.get_repo(f"{owner}/{repo_name}")
+
+        # The API returns releases newest first; drafts are not published releases
+        limit = RELEASE_MODES[release_mode]
+        releases = []
+        for release in gh_repo.get_releases():
+            if release.draft:
+                continue
+            releases.append(release)
+            if limit is not None and len(releases) >= limit:
+                break
+
+        if not releases:
+            logger.info(f"No releases found for {repository.name}")
+            return
+
+        logger.info(f"Downloading {len(releases)} release(s) for {repository.name} (mode: {release_mode})")
+        releases_dir.mkdir(parents=True, exist_ok=True)
+
+        base_headers = {'User-Agent': 'GithubBackup-docker'}
+        if token:
+            base_headers['Authorization'] = f'token {token}'
+
+        for release in releases:
+            tag = release.tag_name or f"release_{release.id}"
+            release_dir = releases_dir / self._safe_filename(tag)
+            release_dir.mkdir(parents=True, exist_ok=True)
+
+            # Source code archive of the tagged commit
+            if release.zipball_url:
+                self._download_file(
+                    release.zipball_url,
+                    release_dir / f"{self._safe_filename(repo_name)}-{self._safe_filename(tag)}-source.zip",
+                    base_headers
+                )
+
+            # Uploaded release assets (binaries, packages, etc.)
+            asset_headers = dict(base_headers, Accept='application/octet-stream')
+            for asset in release.get_assets():
+                self._download_file(asset.url, release_dir / self._safe_filename(asset.name), asset_headers)
+
+            logger.info(f"Downloaded release {tag} for {repository.name}")
+
     def _cleanup_temp_directories(self, repo_backup_dir):
         """Clean up old temporary directories that might be left behind"""
         try:

init_db.py

@@ -20,7 +20,16 @@ def ensure_sqlite_path():
         try:
             # Create tables if they don't exist
             db.create_all()
-
+
+            # Add columns introduced after the initial schema to existing databases
+            from sqlalchemy import inspect, text
+            repo_columns = {col['name'] for col in inspect(db.engine).get_columns('repository')}
+            if 'release_mode' not in repo_columns:
+                with db.engine.begin() as conn:
+                    conn.execute(text("ALTER TABLE repository ADD COLUMN release_mode VARCHAR(10) DEFAULT 'none'"))
+                    conn.execute(text("UPDATE repository SET release_mode = 'none' WHERE release_mode IS NULL"))
+                print("Added release_mode column to repository table")
+
             # Update existing users without theme to use dark theme
             from models import User
             users_without_theme = User.query.filter_by(theme=None).all()

models.py

@@ -4,6 +4,14 @@
 
 db = SQLAlchemy()
 
+# Release backup modes: how many of the most recent releases to include (None = all)
+RELEASE_MODES = {
+    'none': 0,
+    'latest': 1,
+    'last5': 5,
+    'all': None,
+}
+
 class User(UserMixin, db.Model):
     id = db.Column(db.Integer, primary_key=True)
     username = db.Column(db.String(80), unique=True, nullable=False)
@@ -24,6 +32,7 @@ class Repository(db.Model):
     backup_format = db.Column(db.String(20), default='folder')  # folder, zip, tar.gz
     schedule_type = db.Column(db.String(20), default='daily')  # manual, hourly, daily, weekly, monthly, custom
     retention_count = db.Column(db.Integer, default=5)  # Number of backups to keep
+    release_mode = db.Column(db.String(10), default='none')  # none, latest, last5, all
     # Custom schedule fields
     custom_interval = db.Column(db.Integer)  # For custom schedule: interval value (e.g., 3 for "every 3 days")
     custom_unit = db.Column(db.String(10))   # For custom schedule: unit (days, weeks, months)
@@ -35,6 +44,19 @@ class Repository(db.Model):
 
     backup_jobs = db.relationship('BackupJob', backref='repository', lazy=True, cascade='all, delete-orphan')
 
+    @property
+    def owner(self):
+        """GitHub owner (user or organization) taken from the repository URL"""
+        url = (self.url or '').strip()
+        try:
+            if url.startswith('git@'):
+                # git@github.com:owner/repo.git
+                return url.split(':', 1)[1].split('/')[0]
+            # https://github.com/owner/repo(.git)
+            return url.rstrip('/').split('/')[-2]
+        except IndexError:
+            return ''
+
 class BackupJob(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     user_id = db.Column(db.Integer, db.ForeignKey('user.id'), nullable=False)

requirements.txt

@@ -8,7 +8,7 @@ PyGithub==2.10.0
 gitpython==3.1.62
 python-dotenv==1.2.3
 requests==2.34.2
-sqlalchemy==2.0.52
+sqlalchemy==2.0.54
 flask-sqlalchemy==3.1.1
 wtforms==3.2.2
 gunicorn==26.2.0

templates/add_by_username.html

@@ -87,7 +87,20 @@ <h4 class="mb-0">
                             Default number of backup versions to keep for each repository (can be changed per-repo later)
                         </div>
                     </div>
-
+
+                    <div class="mb-3">
+                        <label for="release_mode" class="form-label">Release Files</label>
+                        <select class="form-select" id="release_mode" name="release_mode">
+                            <option value="none" selected>Don't back up releases</option>
+                            <option value="latest">Latest release</option>
+                            <option value="last5">Last 5 releases</option>
+                            <option value="all">All releases</option>
+                        </select>
+                        <div class="form-text">
+                            Default release backup setting for all added repositories (can be changed per-repo later)
+                        </div>
+                    </div>
+
                     <div class="alert alert-info">
                         <i class="fas fa-info-circle"></i>
                         <strong>How this works:</strong>

templates/add_repository.html

@@ -97,7 +97,20 @@ <h4 class="mb-0">
                             Number of backup versions to keep (older backups will be automatically deleted)
                         </div>
                     </div>
-
+
+                    <div class="mb-3">
+                        <label for="release_mode" class="form-label">Release Files</label>
+                        <select class="form-select" id="release_mode" name="release_mode">
+                            <option value="none" selected>Don't back up releases</option>
+                            <option value="latest">Latest release</option>
+                            <option value="last5">Last 5 releases</option>
+                            <option value="all">All releases</option>
+                        </select>
+                        <div class="form-text">
+                            Also include release assets and source archives in each backup (stored in a <code>_releases</code> folder)
+                        </div>
+                    </div>
+
                     <div class="alert alert-info">
                         <i class="fas fa-info-circle"></i>
                         <strong>Note:</strong> 

templates/edit_repository.html

@@ -99,7 +99,21 @@ <h4 class="mb-0">
                             </div>
                         </div>
                     </div>
-
+
+                    <div class="mb-3">
+                        <label for="release_mode" class="form-label">Release Files</label>
+                        {% set release_mode = repository.release_mode or 'none' %}
+                        <select class="form-select" id="release_mode" name="release_mode">
+                            <option value="none" {% if release_mode == 'none' %}selected{% endif %}>Don't back up releases</option>
+                            <option value="latest" {% if release_mode == 'latest' %}selected{% endif %}>Latest release</option>
+                            <option value="last5" {% if release_mode == 'last5' %}selected{% endif %}>Last 5 releases</option>
+                            <option value="all" {% if release_mode == 'all' %}selected{% endif %}>All releases</option>
+                        </select>
+                        <div class="form-text">
+                            Also include release assets and source archives in each backup (stored in a <code>_releases</code> folder)
+                        </div>
+                    </div>
+
                     <div class="mb-3">
                         <div class="form-check">
                             <input class="form-check-input" type="checkbox" id="is_active" name="is_active"