SqlBackup.hpp

// SPDX-License-Identifier: Apache-2.0
#pragma once
 
#include "../Api.hpp"
#include "../SqlConnectInfo.hpp"
#include "../SqlQuery/MigrationPlan.hpp"
#include "../SqlRetryPolicy.hpp"
#include "../SqlSchema.hpp"
 
#include <chrono>
#include <cstdint>
#include <filesystem>
#include <map>
#include <string>
#include <string_view>
#include <vector>
 
/// @defgroup Backup Backup and Restore
/// @brief Parallel chunked database dump and restore, with archive diffing.
///
/// Backups are taken online without a snapshot, so an archive carries no cross-table
/// consistency guarantee.
 
namespace Lightweight::SqlBackup
{
 
/// Compression methods supported for ZIP entries.
///
/// The values correspond to the ZIP compression method IDs used by libzip.
/// Not all methods may be available at runtime depending on how libzip was compiled.
/// Use IsCompressionMethodSupported() to check availability.
// NOLINTNEXTLINE(performance-enum-size) - Values must match libzip ZIP_CM_* constants
enum class CompressionMethod : std::int32_t
{
    Store = 0,   ///< No compression (ZIP_CM_STORE)
    Deflate = 8, ///< Deflate compression (ZIP_CM_DEFLATE) - most compatible
    Bzip2 = 12,  ///< Bzip2 compression (ZIP_CM_BZIP2)
    Lzma = 14,   ///< LZMA compression (ZIP_CM_LZMA)
    Zstd = 93,   ///< Zstandard compression (ZIP_CM_ZSTD)
    Xz = 95,     ///< XZ compression (ZIP_CM_XZ)
};
 
/// @ingroup Backup
/// Configuration for backup operations including compression and chunking.
struct BackupSettings
{
    /// The compression method to use.
    CompressionMethod method = CompressionMethod::Deflate;
 
    /// The compression level (0-9).
    /// - For Deflate: 1 = fastest, 9 = best compression, 6 = default
    /// - For Bzip2: 1-9 (block size in 100k units)
    /// - For Zstd: maps to zstd levels
    /// - For Store: ignored
    std::uint32_t level = 6;
 
    /// The target size in bytes for each chunk before flushing.
    /// Chunks are flushed when the buffer exceeds this size.
    /// Default: 10 MB.
    std::size_t chunkSizeBytes = 10 * 1024 * 1024;
 
    /// Target rows per chunk window. Tables with a single numeric primary key are split into
    /// windows of about this many keys (subject to a per-table window cap) that are backed up
    /// in parallel by multiple workers.
    std::size_t rowsPerChunk = 100'000;
 
    /// Uncompressed bytes each worker accumulates in its private temp archive before sealing it
    /// (the compression of that archive runs in the worker thread at that point, overlapped with
    /// the network-bound fetch). Bounds worker memory at about jobs x workerArchiveBytes and
    /// determines how many temp archives the finalize merge opens. Default: 256 MB.
    std::size_t workerArchiveBytes = 256ULL * 1024 * 1024;
 
    /// If true, only export schema metadata without backing up table data.
    bool schemaOnly = false;
 
    /// Deprecated no-op. Previously bypassed the MS SQL Server single-worker clamp; that clamp has
    /// been removed (all databases now back up multi-threaded), so this flag no longer has any
    /// effect. Retained temporarily to avoid an API/ABI break; pending removal.
    bool forceMssqlConcurrency = false;
};
 
/// @ingroup Backup
/// Configuration for restore operations including memory management.
struct RestoreSettings
{
    /// Batch size for insert operations (rows per batch).
    /// Default: 0 (auto-calculated based on available memory).
    std::size_t batchSize = 0;
 
    /// Maximum rows before an intermediate commit within a chunk.
    /// Helps reduce transaction log / WAL memory accumulation.
    /// Default: 10000. Set to 0 to disable intermediate commits.
    std::size_t maxRowsPerCommit = 10000;
 
    /// Database page cache size in KB (used for SQLite PRAGMA cache_size,
    /// could be extended for other DBMS memory hints).
    /// Default: 65536 (64MB). Set to 0 to use database default.
    std::size_t cacheSizeKB = 65536;
 
    /// Memory limit in bytes (0 = auto-detect from system).
    std::size_t memoryLimitBytes = 0;
 
    /// If true, only recreate schema without importing data.
    bool schemaOnly = false;
};
 
/// Returns available system memory in bytes.
LIGHTWEIGHT_API std::size_t GetAvailableSystemMemory() noexcept;
 
/// Calculates optimal restore settings based on available memory.
///
/// @param availableMemory Available system memory in bytes.
/// @param concurrency Number of concurrent restore workers.
/// @return RestoreSettings optimized for the given memory constraints.
LIGHTWEIGHT_API RestoreSettings CalculateRestoreSettings(std::size_t availableMemory, unsigned concurrency);
 
/// Checks if a compression method is supported by the current libzip installation.
///
/// @param method The compression method to check.
/// @return true if the method is available for both compression and decompression.
LIGHTWEIGHT_API bool IsCompressionMethodSupported(CompressionMethod method) noexcept;
 
/// Returns a list of all compression methods that are supported by the current libzip installation.
LIGHTWEIGHT_API std::vector<CompressionMethod> GetSupportedCompressionMethods() noexcept;
 
/// Returns the human-readable name of a compression method.
LIGHTWEIGHT_API std::string_view CompressionMethodName(CompressionMethod method) noexcept;
 
/// Configuration for retry behavior on transient errors during backup/restore operations.
///
/// An alias for the library-wide @ref SqlRetrySettings: the backup engine was where this policy
/// was first proven out, and it now shares one implementation with the rest of the library rather
/// than keeping a parallel copy. The field names, types and defaults are unchanged, so existing
/// designated-initializer call sites keep compiling verbatim.
///
/// @see SqlRetryPolicy
using RetrySettings = SqlRetrySettings;
 
/// Information about a table being backed up.
struct TableInfo
{
    /// The list of columns in the table in SQL format.
    std::string fields;
 
    /// The list of columns in the table.
    std::vector<bool> isBinaryColumn;
 
    /// The list of columns in the table.
    std::vector<SqlColumnDeclaration> columns;
 
    /// The list of foreign key constraints in the table.
    std::vector<SqlSchema::ForeignKeyConstraint> foreignKeys;
 
    /// The indexes on the table (excluding primary key index).
    std::vector<SqlSchema::IndexDefinition> indexes;
 
    /// The number of rows in the table.
    size_t rowCount = 0;
};
 
/// @ingroup Backup
/// Progress information for backup/restore operations status updates.
struct Progress
{
    /// The state of an individual backup/restore operation.
    enum class State : std::uint8_t
    {
        Started,
        InProgress,
        Finished,
        Error,
        Warning
    };
 
    /// The state of an individual backup/restore operation.
    State state {};
 
    /// The name of the table being backed up / restored.
    std::string tableName;
 
    /// The current number of rows processed.
    size_t currentRows {};
 
    /// The total number of rows to be processed, if known.
    std::optional<size_t> totalRows;
 
    /// A message associated with the progress update.
    std::string message;
};
 
/// @ingroup Backup
/// The interface for progress updates.
struct ProgressManager
{
    virtual ~ProgressManager() = default;
    /// Default constructor.
    ProgressManager() = default;
    /// Default copy constructor.
    ProgressManager(ProgressManager const&) = default;
    /// Default copy assignment operator.
    ProgressManager& operator=(ProgressManager const&) = default;
    /// Default move constructor.
    ProgressManager(ProgressManager&&) = default;
    /// Default move assignment operator.
    ProgressManager& operator=(ProgressManager&&) = default;
 
    /// Gets called when the progress of an individual backup/restore operation changes.
    virtual void Update(Progress const& p) = 0;
 
    /// Gets called when all backup/restore operations are finished.
    virtual void AllDone() = 0;
 
    /// Sets the maximum length of a table name.
    /// This is used to align the output of the progress manager.
    virtual void SetMaxTableNameLength(size_t /*len*/) {}
 
    /// Returns the number of errors encountered during the operation.
    [[nodiscard]] virtual size_t ErrorCount() const noexcept
    {
        return 0;
    }
 
    /// Sets the total number of items to be processed (for ETA calculation).
    /// @param totalItems Total number of items (rows) to process across all tables.
    virtual void SetTotalItems(size_t totalItems)
    {
        (void) totalItems;
    }
 
    /// Adds to the total number of items for progressive ETA calculation.
    /// This is called as row counts become available during parallel counting.
    /// @param additionalItems Number of additional items (rows) to add to the total.
    virtual void AddTotalItems(size_t additionalItems)
    {
        (void) additionalItems;
    }
 
    /// Called when items are processed (for rate and ETA calculation).
    /// @param count Number of items (rows) just processed.
    virtual void OnItemsProcessed(size_t count)
    {
        (void) count;
    }
 
    /// Announces how many tables the operation will process, once, before any
    /// per-table `Update()` is emitted.
    ///
    /// Backup knows the complete table set after its schema scan and Restore
    /// after reading the archive manifest, i.e. both know the denominator before
    /// any data moves. Without this hook a consumer can only count the distinct
    /// table names it has seen so far, so a progress readout of the form
    /// "processed / total" has a total that climbs as tables are discovered —
    /// the GUI's per-table panel showed "12 / 12" then "12 / 47" then "12 / 700"
    /// on the same run, which reads as the job getting bigger rather than as
    /// progress being made.
    ///
    /// @param totalTables Number of tables that will be processed.
    virtual void SetTotalTables(size_t totalTables)
    {
        (void) totalTables;
    }
};
 
/// @ingroup Backup
/// Base class for progress managers that tracks errors automatically.
class ErrorTrackingProgressManager: public ProgressManager
{
  public:
    void Update(Progress const& progress) override
    {
        if (progress.state == Progress::State::Error)
            ++_errorCount;
    }
 
    [[nodiscard]] size_t ErrorCount() const noexcept override
    {
        return _errorCount;
    }
 
  private:
    size_t _errorCount = 0;
};
 
struct NullProgressManager: ErrorTrackingProgressManager
{
    void Update(Progress const& progress) override
    {
        ErrorTrackingProgressManager::Update(progress);
    }
    void AllDone() override {}
};
 
/// Backs up the database to a file.
///
/// @param outputFile the output file.
/// @param connectionString the connection string used to connect to the database.
/// @param concurrency the number of concurrent jobs.
/// @param progress the progress manager to use for progress updates.
/// @param schema the database schema to backup (optional).
/// @param tableFilter comma-separated table filter patterns (default: "*" for all tables).
///        Supports glob wildcards (* and ?) and schema.table notation.
///        Examples: "Users,Products", "*_log", "dbo.Users", "sales.*"
/// @param retrySettings configuration for retry behavior on transient errors.
/// @param backupSettings configuration for compression method, level, and chunk size.
LIGHTWEIGHT_API void Backup(std::filesystem::path const& outputFile,
                            SqlConnectionString const& connectionString,
                            unsigned concurrency,
                            ProgressManager& progress,
                            std::string const& schema = {},
                            std::string const& tableFilter = "*",
                            RetrySettings const& retrySettings = {},
                            BackupSettings const& backupSettings = {});
 
/// Restores the database from a file.
///
/// @param inputFile the input file.
/// @param connectionString the connection string used to connect to the database.
/// @param concurrency the number of concurrent jobs.
/// @param progress the progress manager to use for progress updates.
/// @param schema the database schema to restore into (optional, overrides backup metadata).
/// @param tableFilter comma-separated table filter patterns (default: "*" for all tables).
///        Supports glob wildcards (* and ?) and schema.table notation.
///        Examples: "Users,Products", "*_log", "dbo.Users", "sales.*"
/// @param retrySettings configuration for retry behavior on transient errors.
LIGHTWEIGHT_API void Restore(std::filesystem::path const& inputFile,
                             SqlConnectionString const& connectionString,
                             unsigned concurrency,
                             ProgressManager& progress,
                             std::string const& schema = {},
                             std::string const& tableFilter = "*",
                             RetrySettings const& retrySettings = {});
 
/// Restores the database from a file with explicit memory management settings.
///
/// @param inputFile the input file.
/// @param connectionString the connection string used to connect to the database.
/// @param concurrency the number of concurrent jobs.
/// @param progress the progress manager to use for progress updates.
/// @param schema the database schema to restore into (optional, overrides backup metadata).
/// @param tableFilter comma-separated table filter patterns (default: "*" for all tables).
/// @param retrySettings configuration for retry behavior on transient errors.
/// @param restoreSettings configuration for memory management during restore.
LIGHTWEIGHT_API void Restore(std::filesystem::path const& inputFile,
                             SqlConnectionString const& connectionString,
                             unsigned concurrency,
                             ProgressManager& progress,
                             std::string const& schema,
                             std::string const& tableFilter,
                             RetrySettings const& retrySettings,
                             RestoreSettings const& restoreSettings);
 
/// Returns a copy of `connectionString` with the values of `PWD=` and
/// `Password=` attributes replaced by `***`.
///
/// The string is parsed attribute-wise (`KEY=VALUE` pairs separated by `;`)
/// following ODBC's quoting rules, so redaction is not fooled by:
/// - brace-quoted values — `PWD={pa;ss}` masks the whole `{...}` group,
///   including the embedded `;`, and a `;` inside any other brace-quoted value
///   (e.g. a driver name) does not start a new attribute;
/// - whitespace after a separator — `...; PWD=secret` is still matched.
///
/// Key matching is case-insensitive and only ever matches a whole attribute
/// name, never a substring of another key (`MyPWD=`) or of a value
/// (`Database=PasswordVault`). Applied by CreateMetadata() so a
/// secretRef-resolved plaintext password never lands in an archive's
/// metadata.json (mirrors dbtool's `list-profiles` redaction).
///
/// @param connectionString Raw ODBC connection string.
/// @return The connection string with password values masked.
[[nodiscard]] LIGHTWEIGHT_API std::string RedactConnectionStringSecrets(std::string_view connectionString);
 
/// Creates the metadata JSON content.
///
/// @param connectionString the connection string used to connect to the database.
/// @param tables the list of tables to backup.
/// @param schema the database schema used for these tables (optional).
LIGHTWEIGHT_API std::string CreateMetadata(SqlConnectionString const& connectionString,
                                           SqlSchema::TableList const& tables,
                                           std::string const& schema = {});
 
/// Parses the metadata JSON content and returns a map of table info.
///
/// @param metadataJson content of the metadata.json file.
LIGHTWEIGHT_API std::map<std::string, TableInfo> ParseSchema(std::string_view metadataJson,
                                                             ProgressManager* progress = nullptr);
 
} // namespace Lightweight::SqlBackup