log_retention.hpp

Go to the documentation of this file.

// SPDX-License-Identifier: AGPL-3.0-or-later
// Copyright 2026 Johnson Ogundeji
// See LICENSE_COMMERCIAL for full terms.
#pragma once
 
#if !defined(SIGNET_ENABLE_COMMERCIAL) || !SIGNET_ENABLE_COMMERCIAL
#error "signet/ai/log_retention.hpp requires SIGNET_ENABLE_COMMERCIAL=ON (AGPL-3.0 commercial tier). See LICENSE_COMMERCIAL."
#endif
 
// ---------------------------------------------------------------------------
// log_retention.hpp -- Log Retention Lifecycle Manager
//
// EU AI Act Art.12(3) requires automatic recording of events over the
// lifetime of the system with defined retention periods (minimum 6 months).
// MiFID II RTS 24 Art.4 requires 5-year retention for order records.
//
// This module provides:
//   - RetentionPolicy: configurable TTL, archival, and size-based rotation
//   - LogRetentionManager: scans log directories and enforces retention
//   - Archival callback for moving expired logs to cold storage
//   - Deletion of logs past the maximum retention period
//   - Compliance summary reporting
//
// Header-only. Part of the signet::forge AI module.
// ---------------------------------------------------------------------------
 
#include "signet/error.hpp"
 
#include <algorithm>
#include <stdexcept>
#include <chrono>
#include <cstdint>
#include <filesystem>
#include <functional>
#include <string>
#include <vector>
 
namespace signet::forge {
 
// ---------------------------------------------------------------------------
// RetentionPolicy
// ---------------------------------------------------------------------------
 
struct RetentionPolicy {
    int64_t min_retention_ns = INT64_C(15778800000000000);  // ~6 months
 
    int64_t max_retention_ns = INT64_C(157788000000000000);  // ~5 years
 
    int64_t archive_after_ns = INT64_C(31557600000000000);  // ~1 year
 
    uint64_t max_active_size_bytes = UINT64_C(10737418240);  // 10 GB
 
    uint64_t max_active_files = 10000;
 
    std::string file_suffix = ".parquet";
 
    bool enable_deletion = false;
 
    bool enable_archival = false;
};
 
// ---------------------------------------------------------------------------
// RetentionSummary
// ---------------------------------------------------------------------------
 
struct RetentionSummary {
    int64_t files_scanned = 0;
 
    int64_t files_active = 0;
 
    int64_t files_archived = 0;
 
    int64_t files_deleted = 0;
 
    int64_t files_failed = 0;
 
    uint64_t active_size_bytes = 0;
 
    uint64_t archived_size_bytes = 0;
 
    uint64_t deleted_size_bytes = 0;
 
    bool dry_run = true;
 
    std::vector<std::string> archived_paths;
 
    std::vector<std::string> deleted_paths;
 
    std::vector<std::string> errors;
};
 
// ---------------------------------------------------------------------------
// LogRetentionManager
// ---------------------------------------------------------------------------
 
class LogRetentionManager {
public:
    using ArchiveCallback = std::function<bool(const std::string& source_path)>;
 
    using PreDeleteCallback = std::function<bool(const std::string& path)>;
 
    explicit LogRetentionManager(RetentionPolicy policy = {})
        : policy_(std::move(policy)) {
        auto gate = commercial::require_feature("LogRetentionManager");
        if (!gate) throw std::runtime_error(gate.error().message);
    }
 
    void set_archive_callback(ArchiveCallback cb) {
        archive_cb_ = std::move(cb);
    }
 
    void set_pre_delete_callback(PreDeleteCallback cb) {
        pre_delete_cb_ = std::move(cb);
    }
 
    [[nodiscard]] inline RetentionSummary enforce(
            const std::string& log_dir, int64_t now_ns) const {
        RetentionSummary summary;
        summary.dry_run = !(policy_.enable_archival || policy_.enable_deletion);
 
        namespace fs = std::filesystem;
 
        if (!fs::exists(log_dir) || !fs::is_directory(log_dir)) {
            summary.errors.push_back("Directory does not exist: " + log_dir);
            return summary;
        }
 
        // Collect matching files with their modification times
        struct FileInfo {
            std::string path;
            int64_t     mod_time_ns;
            uint64_t    size_bytes;
        };
        std::vector<FileInfo> files;
 
        for (const auto& entry : fs::directory_iterator(log_dir)) {
            if (!entry.is_regular_file()) continue;
 
            const auto& path = entry.path();
            if (!path.string().ends_with(policy_.file_suffix)) continue;
 
            std::error_code ec;
            auto mod_time = fs::last_write_time(path, ec);
            if (ec) continue;
 
            // Convert file_time_type to nanoseconds since epoch
            auto sys_time = std::chrono::file_clock::to_sys(mod_time);
            auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(
                sys_time.time_since_epoch()).count();
 
            auto fsize = entry.file_size(ec);
            if (ec) fsize = 0;
 
            files.push_back({path.string(), static_cast<int64_t>(ns), fsize});
        }
 
        // Sort by modification time (oldest first)
        std::sort(files.begin(), files.end(),
                  [](const FileInfo& a, const FileInfo& b) {
                      return a.mod_time_ns < b.mod_time_ns;
                  });
 
        summary.files_scanned = static_cast<int64_t>(files.size());
 
        for (const auto& fi : files) {
            int64_t age_ns = now_ns - fi.mod_time_ns;
 
            if (age_ns > policy_.max_retention_ns) {
                // Past maximum retention — eligible for deletion
                if (policy_.enable_deletion) {
                    bool allow = true;
                    if (pre_delete_cb_) {
                        allow = pre_delete_cb_(fi.path);
                    }
                    if (allow) {
                        std::error_code ec;
                        fs::remove(fi.path, ec);
                        if (ec) {
                            summary.errors.push_back("Failed to delete: " + fi.path
                                                     + " (" + ec.message() + ")");
                            ++summary.files_failed;
                        } else {
                            ++summary.files_deleted;
                            summary.deleted_size_bytes += fi.size_bytes;
                            summary.deleted_paths.push_back(fi.path);
                        }
                    } else {
                        ++summary.files_failed;
                    }
                } else {
                    // Dry-run
                    ++summary.files_deleted;
                    summary.deleted_size_bytes += fi.size_bytes;
                    summary.deleted_paths.push_back(fi.path);
                }
            } else if (age_ns > policy_.archive_after_ns) {
                // Past archival threshold — eligible for archival
                if (policy_.enable_archival && archive_cb_) {
                    bool ok = archive_cb_(fi.path);
                    if (ok) {
                        ++summary.files_archived;
                        summary.archived_size_bytes += fi.size_bytes;
                        summary.archived_paths.push_back(fi.path);
                    } else {
                        summary.errors.push_back("Archive callback failed: " + fi.path);
                        ++summary.files_failed;
                    }
                } else {
                    // Dry-run or no callback
                    ++summary.files_archived;
                    summary.archived_size_bytes += fi.size_bytes;
                    summary.archived_paths.push_back(fi.path);
                }
            } else {
                // Active — within retention window
                ++summary.files_active;
                summary.active_size_bytes += fi.size_bytes;
            }
        }
 
        // Check size-based overflow: if active files exceed max, archive oldest
        if (summary.active_size_bytes > policy_.max_active_size_bytes) {
            // Already sorted oldest-first — the active files are the ones
            // not already in archived/deleted lists. This is informational
            // in the summary for now; the caller can re-invoke with a
            // shorter archive_after_ns to trigger size-based archival.
        }
 
        return summary;
    }
 
    struct FileStatus {
        std::string path;
        int64_t     age_ns;
        uint64_t    size_bytes;
        enum class Classification { ACTIVE, ARCHIVE_ELIGIBLE, DELETE_ELIGIBLE } status;
    };
 
    [[nodiscard]] inline std::vector<FileStatus> list_files(
            const std::string& log_dir, int64_t now_ns) const {
        std::vector<FileStatus> result;
        namespace fs = std::filesystem;
 
        if (!fs::exists(log_dir) || !fs::is_directory(log_dir)) return result;
 
        for (const auto& entry : fs::directory_iterator(log_dir)) {
            if (!entry.is_regular_file()) continue;
            const auto& path = entry.path();
            if (!path.string().ends_with(policy_.file_suffix)) continue;
 
            std::error_code ec;
            auto mod_time = fs::last_write_time(path, ec);
            if (ec) continue;
 
            auto sys_time = std::chrono::file_clock::to_sys(mod_time);
            auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(
                sys_time.time_since_epoch()).count();
 
            auto fsize = entry.file_size(ec);
            if (ec) fsize = 0;
 
            int64_t age = now_ns - static_cast<int64_t>(ns);
 
            FileStatus fs_entry;
            fs_entry.path = path.string();
            fs_entry.age_ns = age;
            fs_entry.size_bytes = fsize;
 
            if (age > policy_.max_retention_ns)
                fs_entry.status = FileStatus::Classification::DELETE_ELIGIBLE;
            else if (age > policy_.archive_after_ns)
                fs_entry.status = FileStatus::Classification::ARCHIVE_ELIGIBLE;
            else
                fs_entry.status = FileStatus::Classification::ACTIVE;
 
            result.push_back(std::move(fs_entry));
        }
 
        // Sort by age (oldest first)
        std::sort(result.begin(), result.end(),
                  [](const FileStatus& a, const FileStatus& b) {
                      return a.age_ns > b.age_ns;
                  });
 
        return result;
    }
 
    [[nodiscard]] const RetentionPolicy& policy() const noexcept { return policy_; }
 
private:
    RetentionPolicy   policy_;
    ArchiveCallback   archive_cb_;
    PreDeleteCallback pre_delete_cb_;
};
 
} // namespace signet::forge