From 58c212b4aeb8c31af6541c8fecb31e1b6123d3c4 Mon Sep 17 00:00:00 2001 From: Illia Komsa Date: Wed, 26 Aug 2026 23:16:50 +0200 Subject: [PATCH 01/10] Support reading the AlphaTheta OneLibrary format OneLibrary, also documented as Device Library Plus, is the successor to the DeviceSQL export.pdb library that rekordbox writes to USB media. A device usually carries both, and a player that understands OneLibrary prefers it. The library is a single SQLite database encrypted with SQLCipher 4. Its page format is implemented in util/crypto, so neither SQLCipher nor OpenSSL is needed to read one: AES-256-CBC, SHA-512, HMAC and PBKDF2 are written from their specifications, and the write-ahead log is folded in before SQLite sees the file, as rekordbox leaves most of a fresh export in the log. Support is read-only. Track metadata, playlists and crates can be read; everything that would change a database throws unsupported_operation. Beat grids, waveforms, hot cues and loops are not in the database at all, as rekordbox leaves them in the ANLZ files beside the music, so onelibrary::library gives the path recorded for a track alongside the key notation and track colour that the format-agnostic interface has nowhere to put. Reading a database needs SQLite 3.36 or newer, built without SQLITE_OMIT_DESERIALIZE. --- CMakeLists.txt | 53 ++ GUIDE.md | 39 +- README.md | 14 +- example/README.md | 33 +- example/onelibrary.cpp | 99 ++ include/djinterop/djinterop.hpp | 1 + include/djinterop/onelibrary/onelibrary.hpp | 151 ++++ src/djinterop/onelibrary/content_table.cpp | 283 ++++++ src/djinterop/onelibrary/content_table.hpp | 124 +++ src/djinterop/onelibrary/crate_impl.cpp | 151 ++++ src/djinterop/onelibrary/crate_impl.hpp | 66 ++ src/djinterop/onelibrary/database_impl.cpp | 214 +++++ src/djinterop/onelibrary/database_impl.hpp | 68 ++ src/djinterop/onelibrary/onelibrary.cpp | 254 ++++++ .../onelibrary/onelibrary_context.hpp | 88 ++ src/djinterop/onelibrary/playlist_impl.cpp | 147 +++ src/djinterop/onelibrary/playlist_impl.hpp | 72 ++ src/djinterop/onelibrary/playlist_table.cpp | 147 +++ src/djinterop/onelibrary/playlist_table.hpp | 87 ++ src/djinterop/onelibrary/track_impl.cpp | 354 ++++++++ src/djinterop/onelibrary/track_impl.hpp | 114 +++ src/djinterop/util/crypto/aes.cpp | 265 ++++++ src/djinterop/util/crypto/aes.hpp | 64 ++ src/djinterop/util/crypto/sha512.cpp | 271 ++++++ src/djinterop/util/crypto/sha512.hpp | 64 ++ src/djinterop/util/crypto/sqlcipher_codec.cpp | 200 ++++ src/djinterop/util/crypto/sqlcipher_codec.hpp | 159 ++++ src/djinterop/util/crypto/sqlcipher_wal.cpp | 231 +++++ src/djinterop/util/crypto/sqlcipher_wal.hpp | 46 + src/djinterop/util/filesystem.cpp | 9 + src/djinterop/util/filesystem.hpp | 1 + .../onelibrary/content_table_test.cpp | 247 +++++ test/djinterop/onelibrary/database_test.cpp | 852 ++++++++++++++++++ .../onelibrary/onelibrary_schema.hpp | 75 ++ .../onelibrary/playlist_table_test.cpp | 208 +++++ test/djinterop/sqlcipher_encryptor.hpp | 134 +++ test/djinterop/util/crypto_test.cpp | 300 ++++++ 37 files changed, 5676 insertions(+), 9 deletions(-) create mode 100644 example/onelibrary.cpp create mode 100644 include/djinterop/onelibrary/onelibrary.hpp create mode 100644 src/djinterop/onelibrary/content_table.cpp create mode 100644 src/djinterop/onelibrary/content_table.hpp create mode 100644 src/djinterop/onelibrary/crate_impl.cpp create mode 100644 src/djinterop/onelibrary/crate_impl.hpp create mode 100644 src/djinterop/onelibrary/database_impl.cpp create mode 100644 src/djinterop/onelibrary/database_impl.hpp create mode 100644 src/djinterop/onelibrary/onelibrary.cpp create mode 100644 src/djinterop/onelibrary/onelibrary_context.hpp create mode 100644 src/djinterop/onelibrary/playlist_impl.cpp create mode 100644 src/djinterop/onelibrary/playlist_impl.hpp create mode 100644 src/djinterop/onelibrary/playlist_table.cpp create mode 100644 src/djinterop/onelibrary/playlist_table.hpp create mode 100644 src/djinterop/onelibrary/track_impl.cpp create mode 100644 src/djinterop/onelibrary/track_impl.hpp create mode 100644 src/djinterop/util/crypto/aes.cpp create mode 100644 src/djinterop/util/crypto/aes.hpp create mode 100644 src/djinterop/util/crypto/sha512.cpp create mode 100644 src/djinterop/util/crypto/sha512.hpp create mode 100644 src/djinterop/util/crypto/sqlcipher_codec.cpp create mode 100644 src/djinterop/util/crypto/sqlcipher_codec.hpp create mode 100644 src/djinterop/util/crypto/sqlcipher_wal.cpp create mode 100644 src/djinterop/util/crypto/sqlcipher_wal.hpp create mode 100644 test/djinterop/onelibrary/content_table_test.cpp create mode 100644 test/djinterop/onelibrary/database_test.cpp create mode 100644 test/djinterop/onelibrary/onelibrary_schema.hpp create mode 100644 test/djinterop/onelibrary/playlist_table_test.cpp create mode 100644 test/djinterop/sqlcipher_encryptor.hpp create mode 100644 test/djinterop/util/crypto_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b9411f7..30a1f90 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,6 +82,7 @@ add_library( include/djinterop/engine/v3/track_table.hpp include/djinterop/exceptions.hpp include/djinterop/musical_key.hpp + include/djinterop/onelibrary/onelibrary.hpp include/djinterop/pad_color.hpp include/djinterop/performance_data.hpp include/djinterop/playlist.hpp @@ -203,10 +204,32 @@ add_library( src/djinterop/impl/playlist_impl.hpp src/djinterop/impl/track_impl.cpp src/djinterop/impl/track_impl.hpp + src/djinterop/onelibrary/content_table.cpp + src/djinterop/onelibrary/content_table.hpp + src/djinterop/onelibrary/crate_impl.cpp + src/djinterop/onelibrary/crate_impl.hpp + src/djinterop/onelibrary/database_impl.cpp + src/djinterop/onelibrary/database_impl.hpp + src/djinterop/onelibrary/onelibrary.cpp + src/djinterop/onelibrary/onelibrary_context.hpp + src/djinterop/onelibrary/playlist_impl.cpp + src/djinterop/onelibrary/playlist_impl.hpp + src/djinterop/onelibrary/playlist_table.cpp + src/djinterop/onelibrary/playlist_table.hpp + src/djinterop/onelibrary/track_impl.cpp + src/djinterop/onelibrary/track_impl.hpp src/djinterop/playlist.cpp src/djinterop/track.cpp src/djinterop/util/chrono.cpp src/djinterop/util/chrono.hpp + src/djinterop/util/crypto/aes.cpp + src/djinterop/util/crypto/aes.hpp + src/djinterop/util/crypto/sha512.cpp + src/djinterop/util/crypto/sha512.hpp + src/djinterop/util/crypto/sqlcipher_codec.cpp + src/djinterop/util/crypto/sqlcipher_codec.hpp + src/djinterop/util/crypto/sqlcipher_wal.cpp + src/djinterop/util/crypto/sqlcipher_wal.hpp src/djinterop/util/filesystem.cpp src/djinterop/util/filesystem.hpp src/djinterop/util/random.cpp @@ -370,6 +393,9 @@ install(FILES include/djinterop/engine/v3/track_data_blob.hpp include/djinterop/engine/v3/track_table.hpp DESTINATION "${DJINTEROP_INSTALL_INCLUDEDIR}/engine/v3") +install(FILES + include/djinterop/onelibrary/onelibrary.hpp + DESTINATION "${DJINTEROP_INSTALL_INCLUDEDIR}/onelibrary") if (UNIX) @@ -420,6 +446,7 @@ if (BUILD_EXAMPLES) add_djinterop_example(engine_prime) add_djinterop_example(engine_library_v2_low_level) + add_djinterop_example(onelibrary) endif() # Unit tests. @@ -444,6 +471,10 @@ if (Boost_FOUND AND Boost_filesystem_FOUND AND Boost_system_FOUND) ${Boost_INCLUDE_DIRS} ${CMAKE_CURRENT_BINARY_DIR}/include include) + # A test that compiles library sources into itself, rather than only + # linking, needs the headers that those sources include. + target_include_directories(${test_executable_name} PRIVATE SYSTEM + $) target_link_libraries(${test_executable_name} PUBLIC DjInterop ${Boost_LIBRARIES}) @@ -465,6 +496,28 @@ if (Boost_FOUND AND Boost_filesystem_FOUND AND Boost_system_FOUND) add_djinterop_test(engine/v3/ performance_data_table_test) add_djinterop_test(engine/v3/ track_table_test) + # Some parts of the library are internal, and its symbols are hidden, so + # tests of them compile those sources into themselves rather than linking. + add_djinterop_test(onelibrary/ content_table_test) + target_sources(onelibrary_content_table_test PRIVATE + src/djinterop/onelibrary/content_table.cpp) + + add_djinterop_test(onelibrary/ database_test) + target_sources(onelibrary_database_test PRIVATE + src/djinterop/util/crypto/aes.cpp + src/djinterop/util/crypto/sha512.cpp + src/djinterop/util/crypto/sqlcipher_codec.cpp + src/djinterop/util/filesystem.cpp) + + add_djinterop_test(onelibrary/ playlist_table_test) + target_sources(onelibrary_playlist_table_test PRIVATE + src/djinterop/onelibrary/playlist_table.cpp) + + add_djinterop_test(util/ crypto_test) + target_sources(util_crypto_test PRIVATE + src/djinterop/util/crypto/aes.cpp + src/djinterop/util/crypto/sha512.cpp + src/djinterop/util/crypto/sqlcipher_codec.cpp) else() message(STATUS "Unit tests not available, as the Boost.Filesystem and Boost.System " diff --git a/GUIDE.md b/GUIDE.md index 875ccc7..6cd73d8 100644 --- a/GUIDE.md +++ b/GUIDE.md @@ -73,9 +73,42 @@ As such, in order to create a new library or load an existing library with the intention of operating on it using the high-level API, it is always necessary to start with format-specific functions to do so: -| Library Type | Include Path | -|--------------|------------------------------------------| -| Engine | `#include ` | +| Library Type | Include Path | +|--------------|--------------------------------------------------| +| Engine | `#include ` | +| OneLibrary | `#include ` | + +OneLibrary +---------- + +The AlphaTheta OneLibrary format, also documented as Device Library Plus, is +the successor to the DeviceSQL `export.pdb` library that rekordbox wrote to USB +media. A database is loaded by way of `onelibrary::load_database()`, given +either the root directory of a device or the database file itself. A number of +aspects of the format are worth noting: + +* Support is currently read-only, and everything that would change a database + throws `djinterop::unsupported_operation`. +* Beat grids, waveforms, hot cues and loops are not held in the database. + rekordbox leaves them in the ANLZ files that `content.analysisDataFilePath` + points at, and does not populate the `cue` table on export. Those accessors + therefore return nothing rather than throwing. A caller that reads ANLZ + files itself can load the device as an `onelibrary::library`, whose + `analysis_path()` gives the path recorded for a track, relative to the root + of the device; `library::db()` then gives the same database that + `load_database()` would have. +* `onelibrary::library` also reaches the two other things a device carries + that the format-agnostic interface has nowhere to put: `key_name()` gives + the musical key in the notation rekordbox wrote, which may be Camelot and + which `track::key()` cannot represent, and `color_id()` gives the colour the + DJ marked a track with, numbered as `export.pdb` numbers them. +* The format has a single tree that serves as both playlists and crates, so + `playlists_and_crates_are_distinct` is false and the two views show the same + rows. +* A device is read by decrypting it into memory. rekordbox writes the library + in write-ahead-logged mode, and the log has to be folded in before SQLite + sees the file, so reading one needs SQLite 3.36 or newer, built without + `SQLITE_OMIT_DESERIALIZE`. Stable API/ABI diff --git a/README.md b/README.md index 4951cd1..b90219d 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ State of Support ================ The library is currently in development, and not all features are implemented -yet. It currently supports only the Engine Library format. +yet. It supports the Engine Library format, and can read the AlphaTheta +OneLibrary format. What is supported: @@ -27,11 +28,20 @@ The library supports the following firmware and application versions: SC6000/M) may work, but this is currently untested. * Engine DJ Desktop (aka Engine Prime) from 1.0.1 to 4.3.0. +The library also reads the AlphaTheta OneLibrary format, which rekordbox 7 +writes to USB media as `PIONEER/rekordbox/exportLibrary.db`, and which players +from the CDJ-3000X, XDJ-AZ, OPUS-QUAD and OMNIS-DUO onwards read, as does the +CDJ-3000 from firmware 3.15. Track metadata, playlists and crates can be read. +Writing is not supported yet, and beat grids, waveforms, hot cues and loops are +not held in the database at all, as rekordbox leaves them in the ANLZ files +beside it. The format is also documented under the name Device Library Plus. + What is not supported (yet): * Album art * Play history -* DJ record libraries in formats other than Engine Prime +* Writing OneLibrary databases +* DJ record libraries in formats other than Engine Prime and OneLibrary How Do I Use It? ================ diff --git a/example/README.md b/example/README.md index 5f24768..364835e 100644 --- a/example/README.md +++ b/example/README.md @@ -4,9 +4,34 @@ Overview This directory contains small example applications that illustrate the use of `libdjinterop`. -This application can be minimally compiled in isolation with an invocation -similar to the below (adjust for your favourite compiler as appropriate): +| Example | What it does | +|-------------------------------|---------------------------------------------------------------------| +| `engine_prime` | Writes a track, a crate and a playlist to an Engine Prime library. | +| `engine_library_v2_low_level` | Uses the low-level Engine v2 API to work with tables directly. | +| `onelibrary` | Prints the tracks and playlists of an AlphaTheta OneLibrary device. | + +Each application can be minimally compiled in isolation with an invocation +similar to the below (adjust for your favourite compiler as appropriate). The +library needs a C++20 compiler, and its headers refuse to compile under any +older standard: + +```shell +g++ -std=c++20 `pkg-config --cflags djinterop` engine_prime.cpp `pkg-config --libs djinterop` -o engine_prime +``` + +Any of the others is compiled the same way, by name: + +```shell +g++ -std=c++20 `pkg-config --cflags djinterop` onelibrary.cpp `pkg-config --libs djinterop` -o onelibrary +``` + +They are also built by the project itself, as `example_engine_prime` and so +on, when it is configured with `-DBUILD_EXAMPLES=ON`. + +`onelibrary` takes the device to read as its argument, either the root +directory of a device or the `exportLibrary.db` file itself, and optionally a +passphrase: ```shell -g++ -std=c++17 `pkg-config --cflags djinterop` engine_prime.cpp `pkg-config --libs djinterop` -``` \ No newline at end of file +./onelibrary /Volumes/MYUSB +``` diff --git a/example/onelibrary.cpp b/example/onelibrary.cpp new file mode 100644 index 0000000..0ffb9ab --- /dev/null +++ b/example/onelibrary.cpp @@ -0,0 +1,99 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +// Print the contents of an AlphaTheta OneLibrary device. +// +// onelibrary /Volumes/MYUSB [passphrase] +// +// The first argument is the root of the device -- the directory holding +// `PIONEER` -- or the `exportLibrary.db` file itself. The second is the +// passphrase, which defaults to the one rekordbox uses. + +#include +#include + +#include + +namespace +{ +void print_playlist(const djinterop::playlist& pl, int depth) +{ + const std::string indent(static_cast(depth) * 2, ' '); + std::cout << indent << "- " << pl.name() << " (" << pl.tracks().size() + << " tracks)\n"; + + for (auto&& child : pl.children()) + print_playlist(child, depth + 1); +} + +} // anonymous namespace + +int main(int argc, char** argv) +{ + if (argc < 2) + { + std::cerr << "usage: onelibrary [passphrase]\n"; + return 2; + } + + const std::string device = argv[1]; + const std::string passphrase = + argc > 2 ? argv[2] : djinterop::onelibrary::default_passphrase; + + if (!djinterop::onelibrary::database_exists(device)) + { + std::cerr << "No OneLibrary database found in " << device << "\n"; + return 1; + } + + try + { + auto db = djinterop::onelibrary::load_database(device, passphrase); + + std::cout << "Format: " << db.version_name() << "\n" + << "Library: " << db.uuid() << "\n\n"; + + std::cout << "Tracks\n------\n"; + for (auto&& track : db.tracks()) + { + std::cout << track.id() << ". " + << track.title().value_or("(untitled)") << " - " + << track.artist().value_or("(unknown artist)"); + + if (const auto bpm = track.bpm()) + std::cout << " [" << *bpm << " BPM]"; + + if (const auto key = track.key()) + std::cout << " [" << *key << "]"; + + std::cout << "\n " << track.relative_path() << "\n"; + } + + std::cout << "\nPlaylists\n---------\n"; + for (auto&& pl : db.root_playlists()) + print_playlist(pl, 0); + } + catch (const djinterop::unsupported_database& e) + { + // The most likely cause is a passphrase that does not open the + // database, which happens if rekordbox has changed it. + std::cerr << "Could not read the library: " << e.what() << "\n"; + return 1; + } + + return 0; +} diff --git a/include/djinterop/djinterop.hpp b/include/djinterop/djinterop.hpp index f0cfe82..962aab6 100644 --- a/include/djinterop/djinterop.hpp +++ b/include/djinterop/djinterop.hpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include diff --git a/include/djinterop/onelibrary/onelibrary.hpp b/include/djinterop/onelibrary/onelibrary.hpp new file mode 100644 index 0000000..35c7513 --- /dev/null +++ b/include/djinterop/onelibrary/onelibrary.hpp @@ -0,0 +1,151 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once +#ifndef DJINTEROP_ONELIBRARY_ONELIBRARY_HPP +#define DJINTEROP_ONELIBRARY_ONELIBRARY_HPP + +#include +#include +#include + +#include +#include + +/// Support for the AlphaTheta OneLibrary device format, also documented as +/// Device Library Plus. +/// +/// OneLibrary succeeds the DeviceSQL `export.pdb` library that rekordbox wrote +/// to USB media; a device usually carries both, and a player that understands +/// OneLibrary prefers it. The library is one SQLite database encrypted with +/// SQLCipher 4, whose page format is implemented in `util/crypto`, so neither +/// SQLCipher nor OpenSSL is needed to read one. +/// +/// Support here is currently **read-only**. Everything that changes a +/// database throws `djinterop::unsupported_operation`. +namespace djinterop::onelibrary +{ +/// Location of the library database within a device, relative to its root. +constexpr const char* database_relative_path = + "PIONEER/rekordbox/exportLibrary.db"; + +/// The passphrase with which rekordbox encrypts every `exportLibrary.db`. +/// +/// It is the same on every installation, and depends on neither licence nor +/// machine, so any player can read any device. Pass a different one to +/// `load_database` if a future release of rekordbox changes it. +constexpr const char* default_passphrase = + "r8gddnr4k847830ar6cqzbkk0el6qytmb3trbbx805jm74vez64i5o8fnrqryqls"; + +/// Test whether a OneLibrary database exists in a given location. +/// +/// \param path Either the root directory of a device, or the database file. +/// \return Returns true if a database is present. +bool DJINTEROP_PUBLIC database_exists(const std::string& path); + +/// Load a OneLibrary database. +/// +/// \param path Either the root directory of a device, such as the mount point +/// of a USB drive, or the `exportLibrary.db` file itself. +/// \param passphrase Passphrase with which the database is encrypted. +/// \return Returns the loaded database. +/// \throws database_not_found If no database is present at the given path. +/// \throws unsupported_database If the passphrase does not open the database, +/// or the SQLite in use is older than 3.36, which +/// is the oldest that can read one. +/// \throws database_inconsistency If the database does not hold the tables +/// that a OneLibrary database is expected to. +database DJINTEROP_PUBLIC load_database( + const std::string& path, + const std::string& passphrase = default_passphrase); + +/// State shared by everything belonging to one loaded database. +struct onelibrary_context; + +/// A loaded OneLibrary device, with access to the parts of the format that +/// the format-agnostic `database` interface has nowhere to put. +/// +/// A device is loaded once and read many times: decryption derives a key, +/// which is deliberately expensive, and then holds the whole database in +/// memory, so a caller that wants both the unified interface and the extras +/// below should load a `library` and take `db()` from it rather than also +/// calling `load_database`. +class DJINTEROP_PUBLIC library +{ +public: + /// Load the database on a device. + /// + /// Arguments and exceptions are those of `load_database`. + explicit library( + const std::string& path, + const std::string& passphrase = default_passphrase); + + /// The device, through the format-agnostic interface. + [[nodiscard]] database db() const; + + /// Root directory of the device, to which every path is relative. + /// + /// This is the directory that was loaded, and not the directory the + /// database file itself sits in. + [[nodiscard]] const std::string& directory() const; + + /// Path of the ANLZ analysis data for a track, relative to `directory()`. + /// + /// rekordbox keeps the beatgrid, cues, loops and waveforms out of the + /// database and in a set of files beside the music, which this names: the + /// `.DAT` file given here, and the siblings that differ from it only in + /// extension. `.EXT` holds the colour waveform and the cues beyond the + /// first three, and `.2EX` the waveform that a touch display draws. + /// + /// \param track_id Identifier of the track, as `track::id()` gives it. + /// \return Returns the path, or no value if the track is not there or + /// carries no analysis data. + [[nodiscard]] std::optional analysis_path( + int64_t track_id) const; + + /// The musical key of a track, in the notation the device records. + /// + /// `track::key()` gives the key as one of the twenty-four this library + /// knows, which loses both the notation rekordbox wrote and any key whose + /// notation is not recognised. This gives back what the device holds, + /// such as `F#m` under the classic setting or `8A` under the Camelot one, + /// for a caller that would rather read it itself. + /// + /// \param track_id Identifier of the track, as `track::id()` gives it. + /// \return Returns the notation, or no value if the track is not there or + /// carries no key. + [[nodiscard]] std::optional key_name(int64_t track_id) const; + + /// The colour a track is marked with, as rekordbox enumerates them. + /// + /// The colours are the eight that rekordbox offers, numbered as the + /// `export.pdb` library beside the database numbers them: one for pink, + /// two red, three orange, four yellow, five green, six aqua, seven blue + /// and eight purple. + /// + /// \param track_id Identifier of the track, as `track::id()` gives it. + /// \return Returns the colour, or no value if the track is not there or + /// is not marked with one. + [[nodiscard]] std::optional color_id(int64_t track_id) const; + +private: + std::shared_ptr context_; +}; + +} // namespace djinterop::onelibrary + +#endif // DJINTEROP_ONELIBRARY_ONELIBRARY_HPP diff --git a/src/djinterop/onelibrary/content_table.cpp b/src/djinterop/onelibrary/content_table.cpp new file mode 100644 index 0000000..7e5fe02 --- /dev/null +++ b/src/djinterop/onelibrary/content_table.cpp @@ -0,0 +1,283 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#include "content_table.hpp" + +#include +#include +#include +#include +#include + +namespace djinterop::onelibrary +{ +namespace +{ +/// Every column the row structure needs, in the order it reads them back. +/// +/// Nothing enforces that a lookup reference resolves, so each is joined +/// outwards. +constexpr const char* select_columns = + "SELECT c.content_id, c.title, artist.name, composer.name, album.name, " + "genre.name, label.name, \"key\".name, c.djComment, c.bpmx100, c.length, " + "c.trackNo, c.releaseYear, c.rating, c.path, c.fileSize, " + "c.bitrate, c.samplingRate " + "FROM content AS c " + "LEFT JOIN artist AS artist ON artist.artist_id = c.artist_id_artist " + "LEFT JOIN artist AS composer " + "ON composer.artist_id = c.artist_id_composer " + "LEFT JOIN album ON album.album_id = c.album_id " + "LEFT JOIN genre ON genre.genre_id = c.genre_id " + "LEFT JOIN label ON label.label_id = c.label_id " + // `key` is quoted throughout, as it is also a SQL keyword. + "LEFT JOIN \"key\" ON \"key\".key_id = c.key_id "; + +/// Treat a column that is present but empty as absent: rekordbox writes an +/// empty string for metadata a track does not carry. +std::optional non_empty(std::optional value) +{ + if (value.has_value() && value->empty()) + return std::nullopt; + + return value; +} + +/// The offset in semitones of a note letter above C, if it is one. +constexpr std::optional semitones_above_c(char note) +{ + switch (note) + { + case 'C': return 0; + case 'D': return 2; + case 'E': return 4; + case 'F': return 5; + case 'G': return 7; + case 'A': return 9; + case 'B': return 11; + default: return std::nullopt; + } +} + +/// Major keys, indexed by semitones above C. +constexpr musical_key major_keys[12] = { + musical_key::c_major, musical_key::d_flat_major, + musical_key::d_major, musical_key::e_flat_major, + musical_key::e_major, musical_key::f_major, + musical_key::f_sharp_major, musical_key::g_major, + musical_key::a_flat_major, musical_key::a_major, + musical_key::b_flat_major, musical_key::b_major}; + +/// Minor keys, indexed by semitones above C. +constexpr musical_key minor_keys[12] = { + musical_key::c_minor, musical_key::d_flat_minor, + musical_key::d_minor, musical_key::e_flat_minor, + musical_key::e_minor, musical_key::f_minor, + musical_key::f_sharp_minor, musical_key::g_minor, + musical_key::a_flat_minor, musical_key::a_minor, + musical_key::b_flat_minor, musical_key::b_minor}; + +} // anonymous namespace + +std::optional parse_musical_key(const std::string& name) +{ + // Notation is a note letter, an optional accidental, and an optional `m` + // for a minor key: `C`, `F#m`, `Bb`. Both the ASCII and the typographic + // accidentals are accepted. + size_t position = 0; + if (position >= name.size()) + return std::nullopt; + + const auto note = semitones_above_c(static_cast( + std::toupper(static_cast(name[position])))); + if (!note) + return std::nullopt; + + auto semitone = *note; + ++position; + + // Step over an accidental if one is next. The ASCII and typographic + // spellings mean the same thing and differ only in how many bytes they + // occupy, which `strlen` rather than a literal count keeps right. + const auto consume = [&](const char* accidental) + { + const auto length = std::strlen(accidental); + if (name.compare(position, length, accidental) != 0) + return false; + + position += length; + return true; + }; + + if (consume("#") || consume("♯")) // MUSIC SHARP SIGN + semitone += 1; + else if (consume("b") || consume("♭")) // MUSIC FLAT SIGN + semitone -= 1; + + semitone = ((semitone % 12) + 12) % 12; + + const auto remainder = name.substr(position); + if (remainder.empty()) + return major_keys[semitone]; + + if (remainder == "m" || remainder == "M") + return minor_keys[semitone]; + + // Anything else is a notation this library does not know, such as the + // Camelot or Open Key wheels. + return std::nullopt; +} + +content_table::content_table(std::shared_ptr context) : + context_{std::move(context)} +{ +} + +std::optional content_table::get(int64_t id) const +{ + std::optional result; + + // The parameter list has to match `select_columns` exactly. + context_->db << (std::string{select_columns} + "WHERE c.content_id = ?") + << id >> + [&](int64_t row_id, std::optional title, + std::optional artist, + std::optional composer, + std::optional album, std::optional genre, + std::optional label, std::optional key, + std::optional comment, std::optional bpm_x100, + std::optional length_seconds, + std::optional track_number, + std::optional release_year, + std::optional rating_stars, + std::optional path, std::optional file_size, + std::optional bitrate, + std::optional sampling_rate) + { + content_row row; + row.id = row_id; + row.title = non_empty(std::move(title)); + row.artist = non_empty(std::move(artist)); + row.composer = non_empty(std::move(composer)); + row.album = non_empty(std::move(album)); + row.genre = non_empty(std::move(genre)); + row.label = non_empty(std::move(label)); + row.key = non_empty(std::move(key)); + row.comment = non_empty(std::move(comment)); + row.bpm_x100 = bpm_x100; + row.length_seconds = length_seconds; + row.track_number = track_number; + row.release_year = release_year; + row.rating_stars = rating_stars; + row.path = non_empty(std::move(path)); + row.file_size = file_size; + row.bitrate = bitrate; + row.sampling_rate = sampling_rate; + result = std::move(row); + }; + + return result; +} + +std::vector content_table::all_ids() const +{ + return collect_ids( + *context_, "SELECT content_id FROM content ORDER BY content_id"); +} + +std::vector content_table::ids_by_path(const std::string& path) const +{ + const auto qualified = + !path.empty() && path.front() == '/' ? path : "/" + path; + + return collect_ids( + *context_, + "SELECT content_id FROM content WHERE path = ? ORDER BY content_id", + qualified); +} + +bool content_table::exists(int64_t id) const +{ + return any_row( + *context_, "SELECT 1 FROM content WHERE content_id = ? LIMIT 1", id); +} + +track_snapshot to_snapshot(const content_row& row) +{ + track_snapshot snapshot; + + snapshot.title = row.title; + snapshot.artist = row.artist; + snapshot.composer = row.composer; + snapshot.album = row.album; + snapshot.genre = row.genre; + snapshot.publisher = row.label; + snapshot.comment = row.comment; + + if (row.bpm_x100.has_value() && *row.bpm_x100 > 0) + snapshot.bpm = static_cast(*row.bpm_x100) / 100; + + if (row.length_seconds.has_value() && *row.length_seconds > 0) + snapshot.duration = + std::chrono::milliseconds{*row.length_seconds * 1000}; + + if (row.track_number.has_value() && *row.track_number > 0) + snapshot.track_number = static_cast(*row.track_number); + + if (row.release_year.has_value() && *row.release_year > 0) + snapshot.year = static_cast(*row.release_year); + + // djinterop rates a track from zero to one hundred, where rekordbox uses + // whole stars. + if (row.rating_stars.has_value()) + snapshot.rating = + static_cast(std::clamp(*row.rating_stars, 0, 5) * 20); + + if (row.path.has_value()) + { + // Paths are absolute within the device, whereas djinterop wants them + // relative to the directory of the database. + const auto& path = *row.path; + snapshot.relative_path = path.front() == '/' ? path.substr(1) : path; + } + + if (row.file_size.has_value() && *row.file_size > 0) + snapshot.file_bytes = static_cast(*row.file_size); + + if (row.bitrate.has_value() && *row.bitrate > 0) + snapshot.bitrate = static_cast(*row.bitrate); + + if (row.sampling_rate.has_value() && *row.sampling_rate > 0) + { + snapshot.sample_rate = static_cast(*row.sampling_rate); + + // The database records a duration in whole seconds and no sample + // count, so the count can only be recovered to that precision. + if (row.length_seconds.has_value() && *row.length_seconds > 0) + snapshot.sample_count = static_cast( + *row.length_seconds * *row.sampling_rate); + } + + if (row.key.has_value()) + snapshot.key = parse_musical_key(*row.key); + + // Beatgrids, waveforms, hot cues and loops are not in the database: + // rekordbox leaves them in the ANLZ files that `analysisDataFilePath` + // points at, and exports an empty `cue` table. + return snapshot; +} + +} // namespace djinterop::onelibrary diff --git a/src/djinterop/onelibrary/content_table.hpp b/src/djinterop/onelibrary/content_table.hpp new file mode 100644 index 0000000..00e5341 --- /dev/null +++ b/src/djinterop/onelibrary/content_table.hpp @@ -0,0 +1,124 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "onelibrary_context.hpp" + +namespace djinterop::onelibrary +{ +/// One row of the `content` table, with its lookup tables resolved. +/// +/// The schema declares no constraints at all: nothing is `NOT NULL` and no +/// foreign key is enforced, so every field is optional and a lookup reference +/// may point at a row that is not there. +struct content_row +{ + int64_t id = 0; + + std::optional title; + std::optional artist; + std::optional composer; + std::optional album; + std::optional genre; + + /// The record label, which djinterop calls the publisher. + std::optional label; + + /// The musical key, in the notation rekordbox writes, such as `F#m`. + /// + /// Which notation a device carries follows the setting rekordbox exported + /// it under, so it may equally be Camelot, such as `8A`. + std::optional key; + + /// Free-text comment the DJ attached to the track. + std::optional comment; + + /// Tempo in hundredths of a beat per minute: 12400 is 124.00 BPM. + std::optional bpm_x100; + + /// Duration in whole seconds. + /// + /// The published description of the format calls this column + /// milliseconds, which it is not: on an export written by rekordbox, a + /// 320 kbps track of 6,517,615 bytes carries a `length` of 161, and + /// 6517615 * 8 / 320000 is 163 seconds. Every track on that device + /// agrees to within a second, and none agrees on any other reading. + std::optional length_seconds; + + std::optional track_number; + std::optional release_year; + + /// Rating from zero to five stars, and not the 0-255 encoding that the + /// rest of the rekordbox ecosystem uses: an export written by rekordbox + /// holds only 0 and 5, and 5 is not a value that encoding can take. + std::optional rating_stars; + + /// Device-relative POSIX path, such as + /// `/Contents/Artist/Album/Track.mp3`. + std::optional path; + + std::optional file_size; + std::optional bitrate; + std::optional sampling_rate; +}; + +/// Read access to the `content` table and the lookup tables it references. +class content_table +{ +public: + explicit content_table(std::shared_ptr context); + + /// Fetch one row by its identifier. + [[nodiscard]] std::optional get(int64_t id) const; + + /// Fetch the identifiers of every row, ordered. + [[nodiscard]] std::vector all_ids() const; + + /// Fetch the identifiers of rows whose path matches, ordered. + /// + /// Paths in the database are absolute within the device and begin with a + /// separator; a path given without one is matched as though it had one. + [[nodiscard]] std::vector ids_by_path( + const std::string& path) const; + + /// Test whether a row exists. + [[nodiscard]] bool exists(int64_t id) const; + +private: + std::shared_ptr context_; +}; + +/// Build a track snapshot from a content row. +[[nodiscard]] track_snapshot to_snapshot(const content_row& row); + +/// Interpret the key notation that rekordbox writes, such as `F#m` or `Bb`. +/// +/// Returns no value for a notation that is not recognised, rather than +/// guessing. +[[nodiscard]] std::optional parse_musical_key( + const std::string& name); + +} // namespace djinterop::onelibrary diff --git a/src/djinterop/onelibrary/crate_impl.cpp b/src/djinterop/onelibrary/crate_impl.cpp new file mode 100644 index 0000000..3c798b8 --- /dev/null +++ b/src/djinterop/onelibrary/crate_impl.cpp @@ -0,0 +1,151 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#include "crate_impl.hpp" + +#include + +#include +#include +#include + +#include "database_impl.hpp" +#include "playlist_table.hpp" +#include "track_impl.hpp" + +namespace djinterop::onelibrary +{ +crate_impl::crate_impl( + std::shared_ptr context, int64_t id) : + djinterop::crate_impl{id}, context_{std::move(context)} +{ +} + +crate make_crate(std::shared_ptr context, int64_t id) +{ + return crate{std::make_shared(std::move(context), id)}; +} + +bool crate_impl::is_valid() +{ + return playlist_table{context_}.exists(id()); +} + +std::string crate_impl::name() +{ + const auto row = playlist_table{context_}.get(id()); + if (!row) + throw crate_deleted{id()}; + + return row->name; +} + +std::optional crate_impl::parent() +{ + const auto row = playlist_table{context_}.get(id()); + if (!row) + throw crate_deleted{id()}; + + if (!row->parent_id) + return std::nullopt; + + return make_crate(context_, *row->parent_id); +} + +std::vector crate_impl::children() +{ + std::vector results; + for (const auto& child : playlist_table{context_}.child_ids(id())) + results.push_back(make_crate(context_, child)); + + return results; +} + +std::vector crate_impl::descendants() +{ + std::vector results; + for (const auto& descendant : playlist_table{context_}.descendant_ids(id())) + results.push_back(make_crate(context_, descendant)); + + return results; +} + +std::optional crate_impl::sub_crate_by_name(const std::string& name) +{ + const auto child = playlist_table{context_}.find_child(id(), name); + if (!child) + return std::nullopt; + + return make_crate(context_, *child); +} + +std::vector crate_impl::tracks() +{ + std::vector results; + for (const auto& track_id : playlist_table{context_}.track_ids(id())) + results.push_back( + track{std::make_shared(context_, track_id)}); + + return results; +} + +database crate_impl::db() +{ + return database{std::make_shared(context_)}; +} + +void crate_impl::add_track(int64_t) +{ + read_only(); +} + +void crate_impl::add_track(track) +{ + read_only(); +} + +void crate_impl::clear_tracks() +{ + read_only(); +} + +crate crate_impl::create_sub_crate(const std::string&) +{ + read_only(); +} + +crate crate_impl::create_sub_crate_after(const std::string&, const crate&) +{ + read_only(); +} + +void crate_impl::remove_track(track) +{ + read_only(); +} + +void crate_impl::set_name(std::string) +{ + read_only(); +} + +void crate_impl::set_parent(std::optional) +{ + read_only(); +} + +} // namespace djinterop::onelibrary diff --git a/src/djinterop/onelibrary/crate_impl.hpp b/src/djinterop/onelibrary/crate_impl.hpp new file mode 100644 index 0000000..da8c532 --- /dev/null +++ b/src/djinterop/onelibrary/crate_impl.hpp @@ -0,0 +1,66 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "../impl/crate_impl.hpp" +#include "onelibrary_context.hpp" + +namespace djinterop::onelibrary +{ +/// A crate in a OneLibrary database. +/// +/// The format has one tree of playlists and no concept of a crate, so +/// a crate here is the same row as the playlist of the same identifier. The +/// database reports `playlists_and_crates_are_distinct=false`. +class crate_impl : public djinterop::crate_impl +{ +public: + crate_impl(std::shared_ptr context, int64_t id); + + void add_track(int64_t track_id) override; + void add_track(track tr) override; + std::vector children() override; + void clear_tracks() override; + crate create_sub_crate(const std::string& name) override; + crate create_sub_crate_after( + const std::string& name, const crate& after) override; + database db() override; + std::vector descendants() override; + bool is_valid() override; + std::string name() override; + std::optional parent() override; + void remove_track(track tr) override; + std::optional sub_crate_by_name(const std::string& name) override; + void set_name(std::string name) override; + void set_parent(std::optional parent) override; + std::vector tracks() override; + +private: + std::shared_ptr context_; +}; + +/// Wrap a playlist row as a crate. +crate make_crate(std::shared_ptr context, int64_t id); + +} // namespace djinterop::onelibrary diff --git a/src/djinterop/onelibrary/database_impl.cpp b/src/djinterop/onelibrary/database_impl.cpp new file mode 100644 index 0000000..7569613 --- /dev/null +++ b/src/djinterop/onelibrary/database_impl.cpp @@ -0,0 +1,214 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#include "database_impl.hpp" + +#include +#include +#include + +#include +#include + +#include "content_table.hpp" +#include "crate_impl.hpp" +#include "playlist_impl.hpp" +#include "playlist_table.hpp" +#include "track_impl.hpp" + +namespace djinterop::onelibrary +{ +namespace +{ +/// The tables that must be present for a database to be a OneLibrary one. +/// +/// A real export has twenty-two; demanding the ones this library does not read +/// would reject a database that is merely older or newer. +constexpr std::array required_tables{ + "content", "artist", "album", "genre", "label", + "playlist", "playlist_content", "property"}; + +} // anonymous namespace + +database_impl::database_impl(std::shared_ptr context) : + djinterop::database_impl{ + {feature::supports_nested_crates, feature::supports_nested_playlists, + feature::playlists_support_duplicate_tracks}}, + context_{std::move(context)} +{ +} + +std::string database_impl::directory() +{ + return context_->directory; +} + +std::string database_impl::uuid() +{ + // The format has no identifier for the library as such, so the library + // each track was exported from -- constant across one export -- is the + // closest thing available. + std::string result; + context_->db << "SELECT masterDbId FROM content " + "WHERE masterDbId IS NOT NULL LIMIT 1" >> + [&](std::optional master_db_id) + { + if (master_db_id) + result = std::to_string(*master_db_id); + }; + + return result; +} + +std::string database_impl::version_name() +{ + std::string version; + context_->db << "SELECT dbVersion FROM property LIMIT 1" >> + [&](std::optional db_version) + { version = db_version.value_or(std::string{}); }; + + return version.empty() ? "OneLibrary" : "OneLibrary " + version; +} + +void database_impl::verify() +{ + std::set present; + context_->db << "SELECT name FROM sqlite_master WHERE type = 'table'" >> + [&](std::string name) { present.insert(std::move(name)); }; + + for (const auto& table : required_tables) + if (present.count(table) == 0) + throw database_inconsistency{ + std::string{"The table `"} + table + + "` is missing, so this is not a OneLibrary database"}; +} + +std::optional database_impl::track_by_id(int64_t id) +{ + if (!content_table{context_}.exists(id)) + return std::nullopt; + + return track{std::make_shared(context_, id)}; +} + +std::vector database_impl::tracks() +{ + std::vector results; + for (const auto& id : content_table{context_}.all_ids()) + results.push_back(track{std::make_shared(context_, id)}); + + return results; +} + +std::vector database_impl::tracks_by_relative_path( + const std::string& relative_path) +{ + std::vector results; + for (const auto& id : content_table{context_}.ids_by_path(relative_path)) + results.push_back(track{std::make_shared(context_, id)}); + + return results; +} + +std::optional database_impl::crate_by_id(int64_t id) +{ + if (!playlist_table{context_}.exists(id)) + return std::nullopt; + + return make_crate(context_, id); +} + +std::vector database_impl::root_crates() +{ + std::vector results; + for (const auto& id : playlist_table{context_}.root_ids()) + results.push_back(make_crate(context_, id)); + + return results; +} + +std::optional database_impl::root_crate_by_name( + const std::string& name) +{ + const auto id = playlist_table{context_}.find_root(name); + if (!id) + return std::nullopt; + + return make_crate(context_, *id); +} + +std::vector database_impl::root_playlists() +{ + std::vector results; + for (const auto& id : playlist_table{context_}.root_ids()) + results.push_back(make_playlist(context_, id)); + + return results; +} + +std::optional database_impl::root_playlist_by_name( + const std::string& name) +{ + const auto id = playlist_table{context_}.find_root(name); + if (!id) + return std::nullopt; + + return make_playlist(context_, *id); +} + +crate database_impl::create_root_crate(const std::string&) +{ + read_only(); +} + +crate database_impl::create_root_crate_after(const std::string&, const crate&) +{ + read_only(); +} + +playlist database_impl::create_root_playlist(const std::string&) +{ + read_only(); +} + +playlist database_impl::create_root_playlist_after( + const std::string&, const djinterop::playlist_impl&) +{ + read_only(); +} + +track database_impl::create_track(const track_snapshot&) +{ + read_only(); +} + +void database_impl::remove_crate(djinterop::crate) +{ + read_only(); +} + +void database_impl::remove_playlist(const djinterop::playlist_impl&) +{ + read_only(); +} + +void database_impl::remove_track(djinterop::track) +{ + read_only(); +} + +} // namespace djinterop::onelibrary diff --git a/src/djinterop/onelibrary/database_impl.hpp b/src/djinterop/onelibrary/database_impl.hpp new file mode 100644 index 0000000..87840e6 --- /dev/null +++ b/src/djinterop/onelibrary/database_impl.hpp @@ -0,0 +1,68 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "../impl/database_impl.hpp" +#include "onelibrary_context.hpp" + +namespace djinterop::onelibrary +{ +/// A loaded OneLibrary database, presented through the unified interface. +class database_impl : public djinterop::database_impl +{ +public: + explicit database_impl(std::shared_ptr context); + + std::optional crate_by_id(int64_t id) override; + crate create_root_crate(const std::string& name) override; + crate create_root_crate_after( + const std::string& name, const crate& after) override; + playlist create_root_playlist(const std::string& name) override; + playlist create_root_playlist_after( + const std::string& name, + const djinterop::playlist_impl& after) override; + track create_track(const track_snapshot& snapshot) override; + std::string directory() override; + void verify() override; + void remove_crate(djinterop::crate cr) override; + void remove_playlist(const djinterop::playlist_impl& pl) override; + void remove_track(djinterop::track tr) override; + std::vector root_crates() override; + std::optional root_crate_by_name( + const std::string& name) override; + std::vector root_playlists() override; + std::optional root_playlist_by_name( + const std::string& name) override; + std::optional track_by_id(int64_t id) override; + std::vector tracks() override; + std::vector tracks_by_relative_path( + const std::string& relative_path) override; + std::string uuid() override; + std::string version_name() override; + +private: + std::shared_ptr context_; +}; + +} // namespace djinterop::onelibrary diff --git a/src/djinterop/onelibrary/onelibrary.cpp b/src/djinterop/onelibrary/onelibrary.cpp new file mode 100644 index 0000000..8b9edda --- /dev/null +++ b/src/djinterop/onelibrary/onelibrary.cpp @@ -0,0 +1,254 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#include + +#include +#include +#include +#include + +#include + +#include + +#include "../util/crypto/sqlcipher_codec.hpp" +#include "../util/crypto/sqlcipher_wal.hpp" +#include "../util/filesystem.hpp" +#include "database_impl.hpp" +#include "onelibrary_context.hpp" + +namespace djinterop::onelibrary +{ +namespace +{ +struct resolved_location +{ + /// Root directory of the device, to which track paths are relative. + std::string directory; + + /// The database file itself. + std::string database_path; +}; + +/// Work out where the database is, given either a device or a database file. +resolved_location resolve(const std::string& path) +{ + // A path that names the database directly implies its device root, which + // is three levels up: `/PIONEER/rekordbox/exportLibrary.db`. + if (!util::path_is_directory(path)) + { + auto directory = path; + for (int level = 0; level < 3; ++level) + { + const auto separator = directory.find_last_of("/\\"); + if (separator == std::string::npos) + { + // A relative path with nothing above it sits in the working + // directory, which is then the root of the device. + directory = "."; + break; + } + + directory = directory.substr(0, separator); + } + + return resolved_location{directory, path}; + } + + return resolved_location{path, path + "/" + database_relative_path}; +} + +/// Open the database, folding in its write-ahead log. +/// +/// The database is decrypted into a plain SQLite image, which is handed back +/// as an in-memory database. A device is written in WAL mode and checkpointed +/// on eject, which leaves the header declaring the database +/// write-ahead-logged, and SQLite will not open one of those read-only without +/// the log that is no longer there -- so the log has to be folded in here, and +/// the header rewritten, before SQLite ever sees the bytes. +sqlite::database open_database( + const std::string& database_path, const std::string& passphrase) +{ +#if defined(SQLITE_OMIT_DESERIALIZE) || SQLITE_VERSION_NUMBER < 3036000 + throw unsupported_database{ + "The database `" + database_path + + "` needs SQLite 3.36 or newer, built without SQLITE_OMIT_DESERIALIZE, " + "to read"}; +#else + // Key derivation is deliberately expensive, so it is done once here and + // the codec is handed to everything that reads a page. + const auto codec = util::crypto::make_codec_for(database_path, passphrase); + if (!codec) + throw unsupported_database{ + "The file `" + database_path + "` is too small to be a database"}; + + const auto image = + util::crypto::decrypt_database_to_image(database_path, *codec); + + sqlite::database db{":memory:"}; + + // SQLite takes ownership of the buffer and frees it with the connection, + // so it must come from SQLite's own allocator. + auto* buffer = static_cast(sqlite3_malloc64(image.size())); + if (buffer == nullptr) + throw std::bad_alloc{}; + + std::memcpy(buffer, image.data(), image.size()); + + const auto rc = sqlite3_deserialize( + db.connection().get(), "main", buffer, + static_cast(image.size()), + static_cast(image.size()), + SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_READONLY); + if (rc != SQLITE_OK) + throw unsupported_database{ + "The database `" + database_path + + "` could not be read once it had been decrypted"}; + + return db; +#endif +} + +/// Read one column of a track's row, if the row is there and the column set. +/// +/// The columns behind `library` are each read on their own, rather than +/// through `content_table`, whose query joins six lookup tables to build a +/// whole row that none of them needs. +template +std::optional track_column( + onelibrary_context& context, const char* sql, int64_t track_id) +{ + std::optional result; + context.db << sql << track_id >> + [&](std::optional value) { result = std::move(value); }; + + return result; +} + +/// Decrypt the database on a device and check that it is one. +std::shared_ptr load_context( + const std::string& path, const std::string& passphrase) +{ + const auto location = resolve(path); + + if (!util::path_exists(location.database_path)) + throw database_not_found{location.database_path}; + + // Key derivation is deliberately expensive, so the passphrase is not + // tested separately: a wrong one shows up as the database failing to open, + // and is reported as such. + std::shared_ptr context; + try + { + context = std::make_shared( + location.directory, + open_database(location.database_path, passphrase)); + + // Opening a database reads nothing, so touch it here: a wrong + // passphrase would otherwise not be noticed until the first query. + context->db << "SELECT COUNT(*) FROM sqlite_master" >> [](int64_t) {}; + } + catch (const sqlite::sqlite_exception&) + { + throw unsupported_database{ + "The file `" + location.database_path + + "` is not a SQLCipher database that the given passphrase opens"}; + } + catch (const util::crypto::sqlcipher_error&) + { + throw unsupported_database{ + "The file `" + location.database_path + + "` is not a SQLCipher database that the given passphrase opens"}; + } + + // Fail here, while the caller still has the path in hand, rather than at + // the first query. + database_impl{context}.verify(); + + return context; +} + +} // anonymous namespace + +bool database_exists(const std::string& path) +{ + const auto location = resolve(path); + return util::path_exists(location.database_path); +} + +database load_database(const std::string& path, const std::string& passphrase) +{ + return database{std::make_shared( + load_context(path, passphrase))}; +} + +library::library(const std::string& path, const std::string& passphrase) : + context_{load_context(path, passphrase)} +{ +} + +database library::db() const +{ + return database{std::make_shared(context_)}; +} + +const std::string& library::directory() const +{ + return context_->directory; +} + +std::optional library::analysis_path(int64_t track_id) const +{ + const auto path = track_column( + *context_, "SELECT analysisDataFilePath FROM content WHERE content_id = ?", + track_id); + if (!path || path->empty()) + return std::nullopt; + + // Paths are absolute within the device, whereas every path this library + // hands out is relative to its root. + return path->front() == '/' ? path->substr(1) : *path; +} + +std::optional library::key_name(int64_t track_id) const +{ + const auto name = track_column( + *context_, + "SELECT \"key\".name FROM content AS c " + // `key` is quoted throughout, as it is also a SQL keyword. + "LEFT JOIN \"key\" ON \"key\".key_id = c.key_id " + "WHERE c.content_id = ?", + track_id); + if (!name || name->empty()) + return std::nullopt; + + return name; +} + +std::optional library::color_id(int64_t track_id) const +{ + const auto id = track_column( + *context_, "SELECT color_id FROM content WHERE content_id = ?", + track_id); + if (!id || *id == 0) + return std::nullopt; + + return static_cast(*id); +} + +} // namespace djinterop::onelibrary diff --git a/src/djinterop/onelibrary/onelibrary_context.hpp b/src/djinterop/onelibrary/onelibrary_context.hpp new file mode 100644 index 0000000..1379967 --- /dev/null +++ b/src/djinterop/onelibrary/onelibrary_context.hpp @@ -0,0 +1,88 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include + +namespace djinterop::onelibrary +{ +/// State shared by everything belonging to one loaded OneLibrary database. +struct onelibrary_context +{ + onelibrary_context(std::string directory, sqlite::database db) : + directory{std::move(directory)}, db{std::move(db)} + { + } + + /// Root directory of the device. + /// + /// Track paths in the database are relative to this, not to the directory + /// the database file itself sits in. + const std::string directory; + + /// The database, decrypted from the device into memory. + sqlite::database db; +}; + +/// Refuse an operation that would change the database. +[[noreturn]] inline void read_only() +{ + throw unsupported_operation{ + "OneLibrary databases are currently read-only in libdjinterop"}; +} + +/// Run a query whose rows are each a single identifier. +template +std::vector collect_ids( + onelibrary_context& context, const char* sql, const Args&... args) +{ + std::vector results; + auto query = context.db << sql; + ((query << args), ...); + query >> [&](int64_t id) { results.push_back(id); }; + return results; +} + +/// Run a query whose rows are each a single identifier, and take the first. +template +std::optional first_id( + onelibrary_context& context, const char* sql, const Args&... args) +{ + std::optional result; + auto query = context.db << sql; + ((query << args), ...); + query >> [&](int64_t id) { result = id; }; + return result; +} + +/// Test whether a query matches any row at all. +template +bool any_row(onelibrary_context& context, const char* sql, const Args&... args) +{ + return first_id(context, sql, args...).has_value(); +} + +} // namespace djinterop::onelibrary diff --git a/src/djinterop/onelibrary/playlist_impl.cpp b/src/djinterop/onelibrary/playlist_impl.cpp new file mode 100644 index 0000000..c7da06b --- /dev/null +++ b/src/djinterop/onelibrary/playlist_impl.cpp @@ -0,0 +1,147 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#include "playlist_impl.hpp" + +#include + +#include +#include +#include + +#include "database_impl.hpp" +#include "playlist_table.hpp" +#include "track_impl.hpp" + +namespace djinterop::onelibrary +{ +playlist_impl::playlist_impl( + std::shared_ptr context, int64_t id) : + context_{std::move(context)}, id_{id} +{ +} + +playlist make_playlist(std::shared_ptr context, int64_t id) +{ + return playlist{std::make_shared(std::move(context), id)}; +} + +std::string playlist_impl::name() const +{ + const auto row = playlist_table{context_}.get(id_); + if (!row) + throw playlist_deleted{id_}; + + return row->name; +} + +std::optional playlist_impl::parent() +{ + const auto row = playlist_table{context_}.get(id_); + if (!row) + throw playlist_deleted{id_}; + + if (!row->parent_id) + return std::nullopt; + + return make_playlist(context_, *row->parent_id); +} + +std::vector playlist_impl::children() +{ + std::vector results; + for (const auto& child : playlist_table{context_}.child_ids(id_)) + results.push_back(make_playlist(context_, child)); + + return results; +} + +std::optional playlist_impl::sub_playlist_by_name( + const std::string& name) +{ + const auto child = playlist_table{context_}.find_child(id_, name); + if (!child) + return std::nullopt; + + return make_playlist(context_, *child); +} + +std::vector playlist_impl::tracks() const +{ + std::vector results; + for (const auto& track_id : playlist_table{context_}.track_ids(id_)) + results.push_back( + track{std::make_shared(context_, track_id)}); + + return results; +} + +database playlist_impl::db() const +{ + return database{std::make_shared(context_)}; +} + +bool playlist_impl::operator==(const djinterop::playlist_impl& other) const +{ + const auto* other_impl = dynamic_cast(&other); + return other_impl != nullptr && other_impl->context_ == context_ && + other_impl->id_ == id_; +} + +void playlist_impl::add_track_back(const djinterop::track_impl&) +{ + read_only(); +} + +void playlist_impl::add_track_after( + const djinterop::track_impl&, const djinterop::track_impl&) +{ + read_only(); +} + +void playlist_impl::clear_tracks() +{ + read_only(); +} + +playlist playlist_impl::create_sub_playlist(const std::string&) +{ + read_only(); +} + +playlist playlist_impl::create_sub_playlist_after( + const std::string&, const djinterop::playlist_impl&) +{ + read_only(); +} + +void playlist_impl::remove_track(const djinterop::track_impl&) +{ + read_only(); +} + +void playlist_impl::set_name(const std::string&) +{ + read_only(); +} + +void playlist_impl::set_parent(const djinterop::playlist_impl*) +{ + read_only(); +} + +} // namespace djinterop::onelibrary diff --git a/src/djinterop/onelibrary/playlist_impl.hpp b/src/djinterop/onelibrary/playlist_impl.hpp new file mode 100644 index 0000000..b7d64ea --- /dev/null +++ b/src/djinterop/onelibrary/playlist_impl.hpp @@ -0,0 +1,72 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "../impl/playlist_impl.hpp" +#include "onelibrary_context.hpp" + +namespace djinterop::onelibrary +{ +/// A playlist in a OneLibrary database. +/// +/// The format keeps one tree that serves as both playlists and crates, so this +/// reads the same rows as `crate_impl`. +class playlist_impl : public djinterop::playlist_impl +{ +public: + playlist_impl(std::shared_ptr context, int64_t id); + + [[nodiscard]] int64_t id() const noexcept { return id_; } + + void add_track_back(const djinterop::track_impl& tr) override; + void add_track_after( + const djinterop::track_impl& tr, + const djinterop::track_impl& after) override; + std::vector children() override; + void clear_tracks() override; + playlist create_sub_playlist(const std::string& name) override; + playlist create_sub_playlist_after( + const std::string& name, + const djinterop::playlist_impl& after) override; + [[nodiscard]] database db() const override; + [[nodiscard]] std::string name() const override; + std::optional parent() override; + void remove_track(const djinterop::track_impl& tr) override; + void set_name(const std::string& name) override; + void set_parent(const djinterop::playlist_impl* parent_maybe) override; + std::optional sub_playlist_by_name( + const std::string& name) override; + [[nodiscard]] std::vector tracks() const override; + + bool operator==(const djinterop::playlist_impl& other) const override; + +private: + std::shared_ptr context_; + int64_t id_; +}; + +/// Wrap a playlist row as a playlist. +playlist make_playlist(std::shared_ptr context, int64_t id); + +} // namespace djinterop::onelibrary diff --git a/src/djinterop/onelibrary/playlist_table.cpp b/src/djinterop/onelibrary/playlist_table.cpp new file mode 100644 index 0000000..ae34c78 --- /dev/null +++ b/src/djinterop/onelibrary/playlist_table.cpp @@ -0,0 +1,147 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#include "playlist_table.hpp" + +#include + +namespace djinterop::onelibrary +{ +namespace +{ +// rekordbox marks a root playlist either by leaving the parent unset or by +// writing zero, and nothing in the schema forces one or the other, so both +// spellings have to be matched. +constexpr const char* select_roots = + "SELECT playlist_id FROM playlist " + "WHERE (playlist_id_parent IS NULL OR playlist_id_parent = 0) " + "ORDER BY sequenceNo, playlist_id"; + +constexpr const char* find_root_by_name = + "SELECT playlist_id FROM playlist " + "WHERE (playlist_id_parent IS NULL OR playlist_id_parent = 0) " + "AND name = ? ORDER BY sequenceNo, playlist_id LIMIT 1"; + +/// A parent identifier of zero means the same as none. +std::optional normalise_parent(std::optional parent_id) +{ + if (!parent_id.has_value() || *parent_id == 0) + return std::nullopt; + + return parent_id; +} + +} // anonymous namespace + +playlist_table::playlist_table(std::shared_ptr context) : + context_{std::move(context)} +{ +} + +std::optional playlist_table::get(int64_t id) const +{ + std::optional result; + + context_->db << "SELECT playlist_id, name, playlist_id_parent, sequenceNo " + "FROM playlist WHERE playlist_id = ?" + << id >> + [&](int64_t row_id, std::optional name, + std::optional parent_id, + std::optional sequence_number) + { + result = playlist_row{ + row_id, name.value_or(std::string{}), normalise_parent(parent_id), + sequence_number}; + }; + + return result; +} + +bool playlist_table::exists(int64_t id) const +{ + return any_row( + *context_, "SELECT 1 FROM playlist WHERE playlist_id = ? LIMIT 1", id); +} + +std::vector playlist_table::root_ids() const +{ + return collect_ids(*context_, select_roots); +} + +std::vector playlist_table::child_ids(int64_t id) const +{ + return collect_ids( + *context_, + "SELECT playlist_id FROM playlist WHERE playlist_id_parent = ? " + "ORDER BY sequenceNo, playlist_id", + id); +} + +std::vector playlist_table::descendant_ids(int64_t id) const +{ + // One recursive query rather than one per node. `depth` keeps the result + // breadth first, and `sequenceNo` keeps siblings in their own order. + return collect_ids( + *context_, + "WITH RECURSIVE descendant(playlist_id, sequenceNo, depth) AS (" + "SELECT playlist_id, sequenceNo, 0 FROM playlist " + "WHERE playlist_id_parent = ? " + "UNION ALL " + "SELECT p.playlist_id, p.sequenceNo, descendant.depth + 1 " + "FROM playlist AS p " + "JOIN descendant ON p.playlist_id_parent = descendant.playlist_id) " + "SELECT playlist_id FROM descendant " + "ORDER BY depth, sequenceNo, playlist_id", + id); +} + +std::optional playlist_table::find_root(const std::string& name) const +{ + return first_id(*context_, find_root_by_name, name); +} + +std::optional playlist_table::find_child( + int64_t parent_id, const std::string& name) const +{ + return first_id( + *context_, + "SELECT playlist_id FROM playlist " + "WHERE playlist_id_parent = ? AND name = ? " + "ORDER BY sequenceNo, playlist_id LIMIT 1", + parent_id, name); +} + +std::vector playlist_table::track_ids(int64_t id) const +{ + return collect_ids( + *context_, + "SELECT content_id FROM playlist_content WHERE playlist_id = ? " + "ORDER BY sequenceNo, rowid", + id); +} + +std::vector playlist_table::playlists_containing( + int64_t track_id) const +{ + return collect_ids( + *context_, + "SELECT DISTINCT playlist_id FROM playlist_content " + "WHERE content_id = ? ORDER BY playlist_id", + track_id); +} + +} // namespace djinterop::onelibrary diff --git a/src/djinterop/onelibrary/playlist_table.hpp b/src/djinterop/onelibrary/playlist_table.hpp new file mode 100644 index 0000000..f4e32eb --- /dev/null +++ b/src/djinterop/onelibrary/playlist_table.hpp @@ -0,0 +1,87 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "onelibrary_context.hpp" + +namespace djinterop::onelibrary +{ +/// One row of the `playlist` table. +struct playlist_row +{ + int64_t id = 0; + std::string name; + + /// The playlist this one sits under, if any. + /// + /// A root playlist has either no parent recorded or a parent of zero: the + /// schema enforces no foreign key, and rekordbox writes both. + std::optional parent_id; + + /// Position among siblings, counting from one. + std::optional sequence_number; +}; + +/// Read access to the playlist tree and its membership. +/// +/// OneLibrary has a single tree of playlists, which libdjinterop presents both +/// as playlists and as crates; the two are not distinct in this format. +class playlist_table +{ +public: + explicit playlist_table(std::shared_ptr context); + + [[nodiscard]] std::optional get(int64_t id) const; + + [[nodiscard]] bool exists(int64_t id) const; + + /// Identifiers of the playlists with no parent, in sibling order. + [[nodiscard]] std::vector root_ids() const; + + /// Identifiers of the children of a playlist, in sibling order. + [[nodiscard]] std::vector child_ids(int64_t id) const; + + /// Identifiers of every playlist beneath one, breadth first. + [[nodiscard]] std::vector descendant_ids(int64_t id) const; + + /// Find a root playlist by name. + [[nodiscard]] std::optional find_root( + const std::string& name) const; + + /// Find a child of a playlist by name. + [[nodiscard]] std::optional find_child( + int64_t parent_id, const std::string& name) const; + + /// Identifiers of the tracks in a playlist, in playlist order. + [[nodiscard]] std::vector track_ids(int64_t id) const; + + /// Identifiers of the playlists that hold a given track. + [[nodiscard]] std::vector playlists_containing( + int64_t track_id) const; + +private: + std::shared_ptr context_; +}; + +} // namespace djinterop::onelibrary diff --git a/src/djinterop/onelibrary/track_impl.cpp b/src/djinterop/onelibrary/track_impl.cpp new file mode 100644 index 0000000..3e5117f --- /dev/null +++ b/src/djinterop/onelibrary/track_impl.cpp @@ -0,0 +1,354 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#include "track_impl.hpp" + +#include + +#include +#include + +#include "../util/filesystem.hpp" +#include "content_table.hpp" +#include "crate_impl.hpp" +#include "database_impl.hpp" +#include "playlist_table.hpp" + +namespace djinterop::onelibrary +{ +// Loudness, beatgrids, waveforms, hot cues and loops are absent throughout: +// rekordbox leaves them in the ANLZ files that `analysisDataFilePath` points +// at, and exports an empty `cue` table. Their accessors read as no value +// rather than raising. + +track_impl::track_impl( + std::shared_ptr context, int64_t id) : + djinterop::track_impl{id}, context_{std::move(context)} +{ +} + +track_snapshot track_impl::snapshot() const +{ + const auto row = content_table{context_}.get(id()); + if (!row) + throw track_deleted{id()}; + + return to_snapshot(*row); +} + +bool track_impl::is_valid() +{ + return content_table{context_}.exists(id()); +} + +database track_impl::db() +{ + return database{std::make_shared(context_)}; +} + +std::vector track_impl::containing_crates() +{ + // A crate and a playlist are the same row in this format, so the crates + // containing a track are the playlists that hold it. + std::vector results; + for (const auto& playlist_id : + playlist_table{context_}.playlists_containing(id())) + results.push_back(make_crate(context_, playlist_id)); + + return results; +} + +std::string track_impl::relative_path() +{ + const auto path = snapshot().relative_path; + return path.value_or(std::string{}); +} + +std::string track_impl::filename() +{ + return util::get_filename(relative_path()); +} + +std::string track_impl::file_extension() +{ + return util::get_file_extension(relative_path()).value_or(std::string{}); +} + +std::optional track_impl::album() +{ + return snapshot().album; +} + +std::optional track_impl::artist() +{ + return snapshot().artist; +} + +std::optional track_impl::average_loudness() +{ + return std::nullopt; +} + +std::vector track_impl::beatgrid() +{ + return {}; +} + +std::optional track_impl::bitrate() +{ + return snapshot().bitrate; +} + +std::optional track_impl::bpm() +{ + return snapshot().bpm; +} + +std::optional track_impl::comment() +{ + return snapshot().comment; +} + +std::optional track_impl::composer() +{ + return snapshot().composer; +} + +std::optional track_impl::duration() +{ + return snapshot().duration; +} + +std::optional track_impl::genre() +{ + return snapshot().genre; +} + +std::optional track_impl::hot_cue_at(int) +{ + return std::nullopt; +} + +std::vector> track_impl::hot_cues() +{ + return {}; +} + +std::optional track_impl::key() +{ + return snapshot().key; +} + +std::optional +track_impl::last_played_at() +{ + // The database counts plays but does not record when the last one was. + return std::nullopt; +} + +std::optional track_impl::loop_at(int) +{ + return std::nullopt; +} + +std::vector> track_impl::loops() +{ + return {}; +} + +std::optional track_impl::main_cue() +{ + return std::nullopt; +} + +std::optional track_impl::publisher() +{ + return snapshot().publisher; +} + +std::optional track_impl::rating() +{ + return snapshot().rating; +} + +std::optional track_impl::sample_count() +{ + return snapshot().sample_count; +} + +std::optional track_impl::sample_rate() +{ + return snapshot().sample_rate; +} + +std::optional track_impl::title() +{ + return snapshot().title; +} + +std::optional track_impl::track_number() +{ + return snapshot().track_number; +} + +std::vector track_impl::waveform() +{ + return {}; +} + +std::optional track_impl::year() +{ + return snapshot().year; +} + +void track_impl::update(const track_snapshot&) +{ + read_only(); +} + +void track_impl::set_album(std::optional) +{ + read_only(); +} + +void track_impl::set_artist(std::optional) +{ + read_only(); +} + +void track_impl::set_average_loudness(std::optional) +{ + read_only(); +} + +void track_impl::set_beatgrid(std::vector) +{ + read_only(); +} + +void track_impl::set_bitrate(std::optional) +{ + read_only(); +} + +void track_impl::set_bpm(std::optional) +{ + read_only(); +} + +void track_impl::set_comment(std::optional) +{ + read_only(); +} + +void track_impl::set_composer(std::optional) +{ + read_only(); +} + +void track_impl::set_duration(std::optional) +{ + read_only(); +} + +void track_impl::set_genre(std::optional) +{ + read_only(); +} + +void track_impl::set_hot_cue_at(int, std::optional) +{ + read_only(); +} + +void track_impl::set_hot_cues(std::vector>) +{ + read_only(); +} + +void track_impl::set_key(std::optional) +{ + read_only(); +} + +void track_impl::set_last_played_at( + std::optional) +{ + read_only(); +} + +void track_impl::set_loop_at(int, std::optional) +{ + read_only(); +} + +void track_impl::set_loops(std::vector>) +{ + read_only(); +} + +void track_impl::set_main_cue(std::optional) +{ + read_only(); +} + +void track_impl::set_publisher(std::optional) +{ + read_only(); +} + +void track_impl::set_rating(std::optional) +{ + read_only(); +} + +void track_impl::set_relative_path(std::string) +{ + read_only(); +} + +void track_impl::set_sample_count(std::optional) +{ + read_only(); +} + +void track_impl::set_sample_rate(std::optional) +{ + read_only(); +} + +void track_impl::set_title(std::optional) +{ + read_only(); +} + +void track_impl::set_track_number(std::optional) +{ + read_only(); +} + +void track_impl::set_waveform(std::vector) +{ + read_only(); +} + +void track_impl::set_year(std::optional) +{ + read_only(); +} + +} // namespace djinterop::onelibrary diff --git a/src/djinterop/onelibrary/track_impl.hpp b/src/djinterop/onelibrary/track_impl.hpp new file mode 100644 index 0000000..57a2968 --- /dev/null +++ b/src/djinterop/onelibrary/track_impl.hpp @@ -0,0 +1,114 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "../impl/track_impl.hpp" +#include "onelibrary_context.hpp" + +namespace djinterop::onelibrary +{ +/// A track in a OneLibrary database. +/// +/// Every accessor reads the database afresh rather than caching, so a track +/// handle stays correct for as long as the row is there. Support is currently +/// read-only, and every mutating operation throws. +class track_impl : public djinterop::track_impl +{ +public: + track_impl(std::shared_ptr context, int64_t id); + + track_snapshot snapshot() const override; + void update(const track_snapshot& snapshot) override; + + std::optional album() override; + void set_album(std::optional album) override; + std::optional artist() override; + void set_artist(std::optional artist) override; + std::optional average_loudness() override; + void set_average_loudness(std::optional average_loudness) override; + std::vector beatgrid() override; + void set_beatgrid(std::vector beatgrid) override; + std::optional bitrate() override; + void set_bitrate(std::optional bitrate) override; + std::optional bpm() override; + void set_bpm(std::optional bpm) override; + std::optional comment() override; + void set_comment(std::optional comment) override; + std::optional composer() override; + void set_composer(std::optional composer) override; + std::vector containing_crates() override; + database db() override; + std::optional duration() override; + void set_duration( + std::optional duration) override; + std::string file_extension() override; + std::string filename() override; + std::optional genre() override; + void set_genre(std::optional genre) override; + std::optional hot_cue_at(int index) override; + void set_hot_cue_at(int index, std::optional cue) override; + std::vector> hot_cues() override; + void set_hot_cues(std::vector> cues) override; + bool is_valid() override; + std::optional key() override; + void set_key(std::optional key) override; + std::optional last_played_at() + override; + void set_last_played_at(std::optional + played_at) override; + std::optional loop_at(int index) override; + void set_loop_at(int index, std::optional l) override; + std::vector> loops() override; + void set_loops(std::vector> loops) override; + std::optional main_cue() override; + void set_main_cue(std::optional sample_offset) override; + std::optional publisher() override; + void set_publisher(std::optional publisher) override; + std::optional rating() override; + void set_rating(std::optional rating) override; + std::string relative_path() override; + void set_relative_path(std::string relative_path) override; + std::optional sample_count() override; + void set_sample_count( + std::optional sample_count) override; + std::optional sample_rate() override; + void set_sample_rate(std::optional sample_rate) override; + std::optional title() override; + void set_title(std::optional title) override; + std::optional track_number() override; + void set_track_number(std::optional track_number) override; + std::vector waveform() override; + void set_waveform(std::vector waveform) override; + std::optional year() override; + void set_year(std::optional year) override; + +private: + std::shared_ptr context_; +}; + +} // namespace djinterop::onelibrary diff --git a/src/djinterop/util/crypto/aes.cpp b/src/djinterop/util/crypto/aes.cpp new file mode 100644 index 0000000..d010edc --- /dev/null +++ b/src/djinterop/util/crypto/aes.cpp @@ -0,0 +1,265 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#include "aes.hpp" + +#include + +namespace djinterop::util::crypto +{ +namespace +{ +constexpr int rounds = 14; +constexpr int key_words = 8; + +inline uint8_t rotl8(uint8_t x, unsigned shift) noexcept +{ + return static_cast((x << shift) | (x >> (8 - shift))); +} + +/// Multiply by x in GF(2^8) modulo the AES polynomial. +inline uint8_t xtime(uint8_t x) noexcept +{ + return static_cast((x << 1) ^ ((x & 0x80) ? 0x1b : 0x00)); +} + +/// Multiply two elements of GF(2^8) modulo the AES polynomial. +inline uint8_t gmul(uint8_t a, uint8_t b) noexcept +{ + uint8_t result = 0; + while (b != 0) + { + if (b & 1) + result ^= a; + a = xtime(a); + b = static_cast(b >> 1); + } + return result; +} + +/// The AES substitution box and its inverse. +/// +/// These are derived rather than tabulated: each entry is the affine transform +/// of the multiplicative inverse in GF(2^8), which the standard walk over +/// p = 3^i, q = 3^-i enumerates in a single pass. +struct substitution_tables +{ + uint8_t forward[256]; + uint8_t inverse[256]; + + substitution_tables() noexcept : forward{}, inverse{} + { + uint8_t p = 1; + uint8_t q = 1; + do + { + p = static_cast(p ^ (p << 1) ^ ((p & 0x80) ? 0x1b : 0)); + + // q = q / 3, i.e. q multiplied by the inverse of 3. + q ^= static_cast(q << 1); + q ^= static_cast(q << 2); + q ^= static_cast(q << 4); + if (q & 0x80) + q ^= 0x09; + + const auto value = static_cast( + q ^ rotl8(q, 1) ^ rotl8(q, 2) ^ rotl8(q, 3) ^ rotl8(q, 4) ^ + 0x63); + forward[p] = value; + inverse[value] = p; + } while (p != 1); + + // Zero has no multiplicative inverse; it maps to the affine constant. + forward[0] = 0x63; + inverse[0x63] = 0x00; + } +}; + +const substitution_tables& tables() noexcept +{ + static const substitution_tables instance; + return instance; +} + +} // anonymous namespace + +aes256_cbc::aes256_cbc(const uint8_t* key) noexcept : round_keys_{} +{ + const auto& sbox = tables().forward; + + std::memcpy(round_keys_.data(), key, aes256_key_length); + + uint8_t rcon = 1; + for (int word = key_words; word < 4 * (rounds + 1); ++word) + { + uint8_t temp[4]; + std::memcpy(temp, round_keys_.data() + (4 * (word - 1)), 4); + + if (word % key_words == 0) + { + // Rotate, substitute, and add the round constant. + const auto first = temp[0]; + temp[0] = static_cast(sbox[temp[1]] ^ rcon); + temp[1] = sbox[temp[2]]; + temp[2] = sbox[temp[3]]; + temp[3] = sbox[first]; + rcon = xtime(rcon); + } + else if (word % key_words == 4) + { + for (auto& byte : temp) + byte = sbox[byte]; + } + + for (int i = 0; i < 4; ++i) + { + round_keys_[(4 * word) + i] = + round_keys_[(4 * (word - key_words)) + i] ^ temp[i]; + } + } +} + +void aes256_cbc::encrypt_block( + const uint8_t* input, uint8_t* output) const noexcept +{ + const auto& sbox = tables().forward; + + uint8_t state[16]; + for (int i = 0; i < 16; ++i) + state[i] = input[i] ^ round_keys_[i]; + + for (int round = 1; round <= rounds; ++round) + { + for (auto& byte : state) + byte = sbox[byte]; + + // ShiftRows: the state is column-major, so row r is bytes r, r+4, ... + uint8_t shifted[16]; + for (int column = 0; column < 4; ++column) + for (int row = 0; row < 4; ++row) + shifted[(4 * column) + row] = + state[(4 * ((column + row) % 4)) + row]; + std::memcpy(state, shifted, 16); + + if (round != rounds) + { + for (int column = 0; column < 4; ++column) + { + auto* c = state + (4 * column); + const auto sum = + static_cast(c[0] ^ c[1] ^ c[2] ^ c[3]); + const auto c0 = c[0]; + c[0] ^= static_cast(sum ^ xtime(c[0] ^ c[1])); + c[1] ^= static_cast(sum ^ xtime(c[1] ^ c[2])); + c[2] ^= static_cast(sum ^ xtime(c[2] ^ c[3])); + c[3] ^= static_cast(sum ^ xtime(c[3] ^ c0)); + } + } + + for (int i = 0; i < 16; ++i) + state[i] ^= round_keys_[(16 * round) + i]; + } + + std::memcpy(output, state, 16); +} + +void aes256_cbc::decrypt_block( + const uint8_t* input, uint8_t* output) const noexcept +{ + const auto& rsbox = tables().inverse; + + uint8_t state[16]; + for (int i = 0; i < 16; ++i) + state[i] = input[i] ^ round_keys_[(16 * rounds) + i]; + + for (int round = rounds - 1; round >= 0; --round) + { + // InvShiftRows. + uint8_t shifted[16]; + for (int column = 0; column < 4; ++column) + for (int row = 0; row < 4; ++row) + shifted[(4 * ((column + row) % 4)) + row] = + state[(4 * column) + row]; + std::memcpy(state, shifted, 16); + + for (auto& byte : state) + byte = rsbox[byte]; + + for (int i = 0; i < 16; ++i) + state[i] ^= round_keys_[(16 * round) + i]; + + if (round != 0) + { + for (int column = 0; column < 4; ++column) + { + auto* c = state + (4 * column); + const uint8_t a0 = c[0], a1 = c[1], a2 = c[2], a3 = c[3]; + c[0] = static_cast( + gmul(a0, 14) ^ gmul(a1, 11) ^ gmul(a2, 13) ^ gmul(a3, 9)); + c[1] = static_cast( + gmul(a0, 9) ^ gmul(a1, 14) ^ gmul(a2, 11) ^ gmul(a3, 13)); + c[2] = static_cast( + gmul(a0, 13) ^ gmul(a1, 9) ^ gmul(a2, 14) ^ gmul(a3, 11)); + c[3] = static_cast( + gmul(a0, 11) ^ gmul(a1, 13) ^ gmul(a2, 9) ^ gmul(a3, 14)); + } + } + } + + std::memcpy(output, state, 16); +} + +void aes256_cbc::encrypt( + const uint8_t* iv, const uint8_t* input, uint8_t* output, + size_t length) const noexcept +{ + uint8_t chain[aes_block_length]; + std::memcpy(chain, iv, aes_block_length); + + for (size_t offset = 0; offset < length; offset += aes_block_length) + { + uint8_t block[aes_block_length]; + for (size_t i = 0; i < aes_block_length; ++i) + block[i] = input[offset + i] ^ chain[i]; + + encrypt_block(block, output + offset); + std::memcpy(chain, output + offset, aes_block_length); + } +} + +void aes256_cbc::decrypt( + const uint8_t* iv, const uint8_t* input, uint8_t* output, + size_t length) const noexcept +{ + uint8_t chain[aes_block_length]; + std::memcpy(chain, iv, aes_block_length); + + for (size_t offset = 0; offset < length; offset += aes_block_length) + { + // Retain the ciphertext before writing, in case output aliases input. + uint8_t next_chain[aes_block_length]; + std::memcpy(next_chain, input + offset, aes_block_length); + + decrypt_block(input + offset, output + offset); + for (size_t i = 0; i < aes_block_length; ++i) + output[offset + i] ^= chain[i]; + + std::memcpy(chain, next_chain, aes_block_length); + } +} + +} // namespace djinterop::util::crypto diff --git a/src/djinterop/util/crypto/aes.hpp b/src/djinterop/util/crypto/aes.hpp new file mode 100644 index 0000000..bfa30eb --- /dev/null +++ b/src/djinterop/util/crypto/aes.hpp @@ -0,0 +1,64 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include +#include +#include + +namespace djinterop::util::crypto +{ +constexpr size_t aes_block_length = 16; +constexpr size_t aes256_key_length = 32; + +/// AES-256 in cipher block chaining mode, as specified by FIPS 197 and +/// NIST SP 800-38A. +/// +/// No padding scheme is applied: input lengths must be a whole number of +/// blocks. SQLCipher pages are always block-aligned by construction, which is +/// why a padding mode is not needed. +class aes256_cbc +{ +public: + /// Construct a cipher for a given key, which must be + /// `aes256_key_length` bytes long. + explicit aes256_cbc(const uint8_t* key) noexcept; + + /// Encrypt `length` bytes from `input` into `output`, which may alias + /// `input`. `length` must be a multiple of the AES block length. + /// The initialisation vector must be `aes_block_length` bytes long. + void encrypt( + const uint8_t* iv, const uint8_t* input, uint8_t* output, + size_t length) const noexcept; + + /// Decrypt `length` bytes from `input` into `output`, which may alias + /// `input`. `length` must be a multiple of the AES block length. + /// The initialisation vector must be `aes_block_length` bytes long. + void decrypt( + const uint8_t* iv, const uint8_t* input, uint8_t* output, + size_t length) const noexcept; + +private: + void encrypt_block(const uint8_t* input, uint8_t* output) const noexcept; + void decrypt_block(const uint8_t* input, uint8_t* output) const noexcept; + + /// Expanded key schedule: 15 round keys of 16 bytes each. + std::array round_keys_; +}; + +} // namespace djinterop::util::crypto diff --git a/src/djinterop/util/crypto/sha512.cpp b/src/djinterop/util/crypto/sha512.cpp new file mode 100644 index 0000000..2947727 --- /dev/null +++ b/src/djinterop/util/crypto/sha512.cpp @@ -0,0 +1,271 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#include "sha512.hpp" + +#include +#include +#include + +namespace djinterop::util::crypto +{ +namespace +{ +constexpr uint64_t k[80] = { + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fULL, + 0xe9b5dba58189dbbcULL, 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, 0xd807aa98a3030242ULL, + 0x12835b0145706fbeULL, 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, 0x9bdc06a725c71235ULL, + 0xc19bf174cf692694ULL, 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, 0x2de92c6f592b0275ULL, + 0x4a7484aa6ea6e483ULL, 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, 0xb00327c898fb213fULL, + 0xbf597fc7beef0ee4ULL, 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, 0x27b70a8546d22ffcULL, + 0x2e1b21385c26c926ULL, 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, 0x81c2c92e47edaee6ULL, + 0x92722c851482353bULL, 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, 0xd192e819d6ef5218ULL, + 0xd69906245565a910ULL, 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, 0x2748774cdf8eeb99ULL, + 0x34b0bcb5e19b48a8ULL, 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, 0x748f82ee5defb2fcULL, + 0x78a5636f43172f60ULL, 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 0xbef9a3f7b2c67915ULL, + 0xc67178f2e372532bULL, 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, 0x06f067aa72176fbaULL, + 0x0a637dc5a2c898a6ULL, 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, 0x3c9ebe0a15c9bebcULL, + 0x431d67c49c100d4cULL, 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL}; + +inline uint64_t rotr(uint64_t x, unsigned n) noexcept +{ + return (x >> n) | (x << (64 - n)); +} + +inline uint64_t load_be64(const uint8_t* p) noexcept +{ + uint64_t v = 0; + for (int i = 0; i < 8; ++i) + v = (v << 8) | p[i]; + return v; +} + +inline void store_be64(uint8_t* p, uint64_t v) noexcept +{ + for (int i = 7; i >= 0; --i) + { + p[i] = static_cast(v & 0xff); + v >>= 8; + } +} + +} // anonymous namespace + +sha512::sha512() noexcept : + state_{0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, 0x3c6ef372fe94f82bULL, + 0xa54ff53a5f1d36f1ULL, 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, + 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL}, + buffer_{}, buffered_{0}, total_length_{0} +{ +} + +void sha512::compress(const uint8_t* block) noexcept +{ + uint64_t w[80]; + for (int i = 0; i < 16; ++i) + w[i] = load_be64(block + (i * 8)); + for (int i = 16; i < 80; ++i) + { + const auto s0 = + rotr(w[i - 15], 1) ^ rotr(w[i - 15], 8) ^ (w[i - 15] >> 7); + const auto s1 = + rotr(w[i - 2], 19) ^ rotr(w[i - 2], 61) ^ (w[i - 2] >> 6); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + + auto a = state_[0], b = state_[1], c = state_[2], d = state_[3]; + auto e = state_[4], f = state_[5], g = state_[6], h = state_[7]; + + for (int i = 0; i < 80; ++i) + { + const auto s1 = rotr(e, 14) ^ rotr(e, 18) ^ rotr(e, 41); + const auto ch = (e & f) ^ (~e & g); + const auto temp1 = h + s1 + ch + k[i] + w[i]; + const auto s0 = rotr(a, 28) ^ rotr(a, 34) ^ rotr(a, 39); + const auto maj = (a & b) ^ (a & c) ^ (b & c); + const auto temp2 = s0 + maj; + + h = g; + g = f; + f = e; + e = d + temp1; + d = c; + c = b; + b = a; + a = temp1 + temp2; + } + + state_[0] += a; + state_[1] += b; + state_[2] += c; + state_[3] += d; + state_[4] += e; + state_[5] += f; + state_[6] += g; + state_[7] += h; +} + +void sha512::update(const uint8_t* data, size_t length) noexcept +{ + total_length_ += length; + + if (buffered_ > 0) + { + const auto take = std::min(length, sha512_block_length - buffered_); + std::memcpy(buffer_.data() + buffered_, data, take); + buffered_ += take; + data += take; + length -= take; + + if (buffered_ < sha512_block_length) + return; + + compress(buffer_.data()); + buffered_ = 0; + } + + while (length >= sha512_block_length) + { + compress(data); + data += sha512_block_length; + length -= sha512_block_length; + } + + std::memcpy(buffer_.data(), data, length); + buffered_ = length; +} + +sha512_digest sha512::finalise() noexcept +{ + // The length field is 128 bits wide, but inputs here are far below 2^64 + // bytes, so the high half is always zero. + const uint64_t bit_length = total_length_ * 8; + + buffer_[buffered_++] = 0x80; + if (buffered_ > sha512_block_length - 16) + { + std::fill(buffer_.begin() + buffered_, buffer_.end(), uint8_t{0}); + compress(buffer_.data()); + buffered_ = 0; + } + + std::fill(buffer_.begin() + buffered_, buffer_.end() - 8, uint8_t{0}); + store_be64(buffer_.data() + sha512_block_length - 8, bit_length); + compress(buffer_.data()); + + sha512_digest digest{}; + for (int i = 0; i < 8; ++i) + store_be64(digest.data() + (i * 8), state_[i]); + return digest; +} + +sha512_digest sha512::hash(const uint8_t* data, size_t length) noexcept +{ + sha512 h; + h.update(data, length); + return h.finalise(); +} + +sha512_digest hmac_sha512( + const uint8_t* key, size_t key_length, const uint8_t* data, + size_t data_length) noexcept +{ + std::array padded{}; + if (key_length > sha512_block_length) + { + const auto digest = sha512::hash(key, key_length); + std::copy(digest.begin(), digest.end(), padded.begin()); + } + else + { + std::copy(key, key + key_length, padded.begin()); + } + + std::array inner_pad{}; + std::array outer_pad{}; + for (size_t i = 0; i < sha512_block_length; ++i) + { + inner_pad[i] = static_cast(padded[i] ^ 0x36); + outer_pad[i] = static_cast(padded[i] ^ 0x5c); + } + + sha512 inner; + inner.update(inner_pad.data(), inner_pad.size()); + inner.update(data, data_length); + const auto inner_digest = inner.finalise(); + + sha512 outer; + outer.update(outer_pad.data(), outer_pad.size()); + outer.update(inner_digest.data(), inner_digest.size()); + return outer.finalise(); +} + +std::vector pbkdf2_hmac_sha512( + const uint8_t* password, size_t password_length, const uint8_t* salt, + size_t salt_length, uint32_t iterations, size_t length) +{ + if (iterations == 0) + throw std::invalid_argument{"PBKDF2 requires at least one iteration"}; + + std::vector output; + output.reserve(length); + + std::vector block; + block.reserve(salt_length + 4); + + for (uint32_t index = 1; output.size() < length; ++index) + { + // U_1 = PRF(password, salt || INT_BE32(index)) + block.assign(salt, salt + salt_length); + block.push_back(static_cast(index >> 24)); + block.push_back(static_cast(index >> 16)); + block.push_back(static_cast(index >> 8)); + block.push_back(static_cast(index)); + + auto u = + hmac_sha512(password, password_length, block.data(), block.size()); + auto accumulator = u; + + for (uint32_t i = 1; i < iterations; ++i) + { + u = hmac_sha512(password, password_length, u.data(), u.size()); + for (size_t j = 0; j < accumulator.size(); ++j) + accumulator[j] ^= u[j]; + } + + const auto take = std::min(accumulator.size(), length - output.size()); + output.insert( + output.end(), accumulator.begin(), accumulator.begin() + take); + } + + return output; +} + +} // namespace djinterop::util::crypto diff --git a/src/djinterop/util/crypto/sha512.hpp b/src/djinterop/util/crypto/sha512.hpp new file mode 100644 index 0000000..b26635d --- /dev/null +++ b/src/djinterop/util/crypto/sha512.hpp @@ -0,0 +1,64 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include +#include +#include +#include + +namespace djinterop::util::crypto +{ +constexpr size_t sha512_digest_length = 64; +constexpr size_t sha512_block_length = 128; + +using sha512_digest = std::array; + +/// Streaming SHA-512, as specified by FIPS 180-4. +class sha512 +{ +public: + sha512() noexcept; + + void update(const uint8_t* data, size_t length) noexcept; + + /// Finalise the hash. The object must not be reused afterwards. + [[nodiscard]] sha512_digest finalise() noexcept; + + static sha512_digest hash(const uint8_t* data, size_t length) noexcept; + +private: + void compress(const uint8_t* block) noexcept; + + std::array state_; + std::array buffer_; + size_t buffered_; + uint64_t total_length_; +}; + +/// Compute HMAC-SHA-512, as specified by RFC 2104. +sha512_digest hmac_sha512( + const uint8_t* key, size_t key_length, const uint8_t* data, + size_t data_length) noexcept; + +/// Derive `length` bytes using PBKDF2-HMAC-SHA-512, as specified by RFC 8018. +std::vector pbkdf2_hmac_sha512( + const uint8_t* password, size_t password_length, const uint8_t* salt, + size_t salt_length, uint32_t iterations, size_t length); + +} // namespace djinterop::util::crypto diff --git a/src/djinterop/util/crypto/sqlcipher_codec.cpp b/src/djinterop/util/crypto/sqlcipher_codec.cpp new file mode 100644 index 0000000..e98b549 --- /dev/null +++ b/src/djinterop/util/crypto/sqlcipher_codec.cpp @@ -0,0 +1,200 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#include "sqlcipher_codec.hpp" + +#include + +#include +#include + +namespace djinterop::util::crypto +{ +namespace +{ +constexpr size_t page_key_length = aes256_key_length; +constexpr size_t hmac_key_length = 32; + +/// Derive the page encryption key from the passphrase and the database salt. +std::vector derive_page_key( + const std::string& passphrase, const sqlcipher_salt& salt, + uint32_t iterations) +{ + return pbkdf2_hmac_sha512( + reinterpret_cast(passphrase.data()), passphrase.size(), + salt.data(), salt.size(), iterations, page_key_length); +} + +/// Derive the HMAC key. +/// +/// It comes from the *page key*, not the passphrase, using the database salt +/// with every byte XORed by 0x3a. +std::vector derive_hmac_key( + const std::vector& page_key, const sqlcipher_salt& salt, + uint32_t iterations) +{ + sqlcipher_salt hmac_salt{}; + std::transform( + salt.begin(), salt.end(), hmac_salt.begin(), + [](uint8_t byte) { return static_cast(byte ^ 0x3a); }); + + return pbkdf2_hmac_sha512( + page_key.data(), page_key.size(), hmac_salt.data(), hmac_salt.size(), + iterations, hmac_key_length); +} + +/// Reject parameters that no SQLCipher database could have, before any time +/// is spent on key derivation. +const sqlcipher_parameters& validated(const sqlcipher_parameters& params) +{ + if (params.reserve < aes_block_length + sha512_digest_length) + throw sqlcipher_error{ + "SQLCipher reserve is too small to hold an IV and an HMAC tag"}; + + if (params.page_size <= params.reserve) + throw sqlcipher_error{ + "SQLCipher page size does not exceed its reserved area"}; + + if (params.payload_size() % aes_block_length != 0) + throw sqlcipher_error{ + "SQLCipher page payload is not a whole number of AES blocks"}; + + return params; +} + +/// Compare two byte sequences without leaking their contents through timing. +bool equal_in_constant_time( + const uint8_t* left, const uint8_t* right, size_t length) noexcept +{ + uint8_t difference = 0; + for (size_t i = 0; i < length; ++i) + difference |= static_cast(left[i] ^ right[i]); + return difference == 0; +} + +} // anonymous namespace + +sqlcipher_codec::sqlcipher_codec( + const std::string& passphrase, const sqlcipher_salt& salt, + const sqlcipher_parameters& params) : + sqlcipher_codec{ + derive_page_key(passphrase, salt, validated(params).kdf_iterations), + salt, params} +{ +} + +sqlcipher_codec::sqlcipher_codec( + const std::vector& page_key, const sqlcipher_salt& salt, + const sqlcipher_parameters& params) : + params_{params}, cipher_{page_key.data()}, + hmac_key_{derive_hmac_key(page_key, salt, params.hmac_kdf_iterations)} +{ +} + +sha512_digest sqlcipher_codec::page_mac( + uint32_t page_number, const uint8_t* ciphertext, size_t length, + const uint8_t* iv) const noexcept +{ + // The tag covers ciphertext || IV || page number, where the page number is + // a little-endian 32-bit integer. + std::vector message; + message.reserve(length + aes_block_length + 4); + message.insert(message.end(), ciphertext, ciphertext + length); + message.insert(message.end(), iv, iv + aes_block_length); + message.push_back(static_cast(page_number)); + message.push_back(static_cast(page_number >> 8)); + message.push_back(static_cast(page_number >> 16)); + message.push_back(static_cast(page_number >> 24)); + + return hmac_sha512( + hmac_key_.data(), hmac_key_.size(), message.data(), message.size()); +} + +bool sqlcipher_codec::page_mac_is_valid( + uint32_t page_number, const uint8_t* encrypted) const noexcept +{ + const auto offset = ciphertext_offset(page_number); + const auto length = params_.payload_size() - offset; + const auto* iv = encrypted + params_.payload_size(); + const auto* tag = iv + aes_block_length; + + const auto expected = page_mac(page_number, encrypted + offset, length, iv); + return equal_in_constant_time(expected.data(), tag, sha512_digest_length); +} + +void sqlcipher_codec::decrypt_page( + uint32_t page_number, const uint8_t* encrypted, uint8_t* decrypted) const +{ + const auto offset = ciphertext_offset(page_number); + const auto length = params_.payload_size() - offset; + const auto* iv = encrypted + params_.payload_size(); + + if (!page_mac_is_valid(page_number, encrypted)) + throw sqlcipher_error{ + "page " + std::to_string(page_number) + + " failed its integrity check: wrong passphrase, corrupt data, or " + "not a SQLCipher database"}; + + if (offset > 0 && decrypted != encrypted) + std::memcpy(decrypted, encrypted, offset); + + cipher_.decrypt(iv, encrypted + offset, decrypted + offset, length); + + // Leave the reserved area zeroed. SQLite never looks at it once the + // header declares a reserve size, and zeroing keeps the IV and tag of the + // encrypted page from lingering in the decrypted image. + std::memset(decrypted + params_.payload_size(), 0, params_.reserve); +} + +namespace +{ +/// Read the salt of a SQLCipher database from its first page. +sqlcipher_salt read_salt(const uint8_t* first_page) noexcept +{ + sqlcipher_salt salt{}; + std::copy(first_page, first_page + salt.size(), salt.begin()); + return salt; +} + +/// Read the first page of a database, or nothing if there is not one. +std::optional> read_first_page( + const std::string& database_path, size_t page_size) +{ + std::vector page(page_size); + std::ifstream file{database_path, std::ios::binary}; + if (!file.read( + reinterpret_cast(page.data()), + static_cast(page.size()))) + return std::nullopt; + + return page; +} + +} // anonymous namespace + +std::optional make_codec_for( + const std::string& database_path, const std::string& passphrase, + const sqlcipher_parameters& params) +{ + const auto page = read_first_page(database_path, params.page_size); + if (!page) + return std::nullopt; + + return sqlcipher_codec{passphrase, read_salt(page->data()), params}; +} + +} // namespace djinterop::util::crypto diff --git a/src/djinterop/util/crypto/sqlcipher_codec.hpp b/src/djinterop/util/crypto/sqlcipher_codec.hpp new file mode 100644 index 0000000..3819a7d --- /dev/null +++ b/src/djinterop/util/crypto/sqlcipher_codec.hpp @@ -0,0 +1,159 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "aes.hpp" +#include "sha512.hpp" + +namespace djinterop::util::crypto +{ +/// Parameters of a SQLCipher database. +/// +/// The defaults are those of SQLCipher 4, which is what the AlphaTheta +/// OneLibrary `exportLibrary.db` is written with. +struct sqlcipher_parameters +{ + /// Size of a database page, in bytes, including the reserved area. + size_t page_size = 4096; + + /// Bytes reserved at the end of each page: a 16-byte initialisation + /// vector followed by a 64-byte HMAC-SHA-512 tag. + size_t reserve = 80; + + /// Number of PBKDF2 iterations used to derive the page key. + uint32_t kdf_iterations = 256000; + + /// Number of PBKDF2 iterations used to derive the HMAC key. The input is + /// already a strong key, so stretching it further would cost time for + /// nothing. + uint32_t hmac_kdf_iterations = 2; + + [[nodiscard]] size_t payload_size() const noexcept + { + return page_size - reserve; + } +}; + +constexpr size_t sqlcipher_salt_length = 16; + +using sqlcipher_salt = std::array; + +/// The bytes that open a plain SQLite file, and which SQLCipher overwrites +/// with the key derivation salt. +/// +/// The two occupy the same 16 bytes, so a decrypted first page becomes a +/// loadable one by copying this over its salt. +constexpr const char sqlite_file_magic[] = "SQLite format 3"; +constexpr size_t sqlite_file_magic_length = sqlcipher_salt_length; + +/// Thrown when a page fails to decrypt, or fails its integrity check. +class sqlcipher_error : public std::runtime_error +{ +public: + explicit sqlcipher_error(const std::string& what) : std::runtime_error{what} + { + } +}; + +/// Decrypts the pages of a SQLCipher database. +/// +/// Each page holds `payload_size()` bytes of ciphertext, a random +/// initialisation vector, and an HMAC-SHA-512 tag over the ciphertext, the +/// vector, and the page number -- which binds a page to its position, so pages +/// cannot be swapped without detection. +/// +/// The first page is special: its leading 16 bytes are the key derivation +/// salt, in the clear where a plain SQLite file has its header magic, so its +/// ciphertext is 16 bytes shorter than that of every other page. +class sqlcipher_codec +{ +public: + /// Construct a codec for a passphrase and the salt of a database. + /// + /// Key derivation is deliberately expensive, so construct a codec once per + /// database rather than once per page. + sqlcipher_codec( + const std::string& passphrase, const sqlcipher_salt& salt, + const sqlcipher_parameters& params = {}); + + [[nodiscard]] const sqlcipher_parameters& parameters() const noexcept + { + return params_; + } + + /// Decrypt one page. + /// + /// `encrypted` and `decrypted` are both `page_size` bytes long and may + /// alias each other. Page numbers are one-based, as in SQLite itself, and + /// the reserved area of the output is zeroed. + /// + /// On page one the salt is left in place, so a caller that needs a + /// loadable SQLite image must overwrite those bytes with the header magic. + void decrypt_page( + uint32_t page_number, const uint8_t* encrypted, + uint8_t* decrypted) const; + +private: + /// Test whether a page carries a valid HMAC tag. + /// + /// A wrong passphrase fails here rather than yielding plausible noise. + [[nodiscard]] bool page_mac_is_valid( + uint32_t page_number, const uint8_t* encrypted) const noexcept; + + /// Adopt an already-derived page key. The HMAC key comes from the page + /// key rather than the passphrase, so the public constructor derives once + /// and delegates here. + sqlcipher_codec( + const std::vector& page_key, const sqlcipher_salt& salt, + const sqlcipher_parameters& params); + + /// Offset within a page at which its ciphertext begins. + [[nodiscard]] size_t ciphertext_offset(uint32_t page_number) const noexcept + { + return page_number == 1 ? sqlcipher_salt_length : 0; + } + + [[nodiscard]] sha512_digest page_mac( + uint32_t page_number, const uint8_t* ciphertext, size_t length, + const uint8_t* iv) const noexcept; + + sqlcipher_parameters params_; + aes256_cbc cipher_; + std::vector hmac_key_; +}; + +/// Build the codec of a database, reading its salt from the file. +/// +/// Returns nothing if the file cannot be read or is shorter than a page. Key +/// derivation is deliberately expensive, so a caller that goes on to read more +/// than one page should build the codec once, here, and keep it. +/// +/// \throws sqlcipher_error If the key cannot be derived. +std::optional make_codec_for( + const std::string& database_path, const std::string& passphrase, + const sqlcipher_parameters& params = {}); + +} // namespace djinterop::util::crypto diff --git a/src/djinterop/util/crypto/sqlcipher_wal.cpp b/src/djinterop/util/crypto/sqlcipher_wal.cpp new file mode 100644 index 0000000..59f3a6e --- /dev/null +++ b/src/djinterop/util/crypto/sqlcipher_wal.cpp @@ -0,0 +1,231 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#include "sqlcipher_wal.hpp" + +#include +#include + +namespace djinterop::util::crypto +{ +namespace +{ +constexpr size_t wal_header_length = 32; +constexpr size_t wal_frame_header_length = 24; + +/// The two magic numbers that open a log, differing in the byte order used +/// for its checksums. +constexpr uint32_t wal_magic_little_endian = 0x377f0682; +constexpr uint32_t wal_magic_big_endian = 0x377f0683; + +/// Offsets of the two version bytes in the SQLite header. +constexpr size_t write_version_offset = 18; +constexpr size_t read_version_offset = 19; +constexpr uint8_t rollback_journal_version = 1; + +uint32_t load_be32(const uint8_t* p) noexcept +{ + return (static_cast(p[0]) << 24) | + (static_cast(p[1]) << 16) | + (static_cast(p[2]) << 8) | static_cast(p[3]); +} + +uint32_t load_le32(const uint8_t* p) noexcept +{ + return (static_cast(p[3]) << 24) | + (static_cast(p[2]) << 16) | + (static_cast(p[1]) << 8) | static_cast(p[0]); +} + +std::vector read_file(const std::string& path) +{ + std::ifstream file{path, std::ios::binary}; + if (!file) + return {}; + + return std::vector{ + std::istreambuf_iterator{file}, std::istreambuf_iterator{}}; +} + +/// The running checksum SQLite keeps over the contents of a log. +/// +/// Each step folds a pair of 32-bit words into the pair of accumulators, so +/// the checksum of a frame depends on every frame before it: a log cannot be +/// truncated or reordered without detection. +struct wal_checksum +{ + uint32_t s0 = 0; + uint32_t s1 = 0; + + void accumulate( + const uint8_t* data, size_t length, bool big_endian) noexcept + { + for (size_t offset = 0; offset + 8 <= length; offset += 8) + { + const auto first = big_endian ? load_be32(data + offset) + : load_le32(data + offset); + const auto second = big_endian ? load_be32(data + offset + 4) + : load_le32(data + offset + 4); + s0 += first + s1; + s1 += second + s0; + } + } + + /// Compare against a stored pair of checksums. + /// + /// The byte order of the magic number governs how the *contents* are read + /// while accumulating, but the two stored values are header fields, and + /// every header field in a log is big-endian. + [[nodiscard]] bool matches(const uint8_t* expected) const noexcept + { + return s0 == load_be32(expected) && s1 == load_be32(expected + 4); + } +}; + +} // anonymous namespace + +std::vector decrypt_database_to_image( + const std::string& database_path, const sqlcipher_codec& codec) +{ + const auto& params = codec.parameters(); + auto encrypted = read_file(database_path); + if (encrypted.size() < params.page_size) + throw sqlcipher_error{ + "`" + database_path + "` is too small to be a database"}; + + if (encrypted.size() % params.page_size != 0) + throw sqlcipher_error{ + "`" + database_path + "` is not a whole number of pages"}; + + const auto page_size = params.page_size; + const auto page_count = encrypted.size() / page_size; + + std::vector image(encrypted.size()); + for (size_t index = 0; index < page_count; ++index) + { + codec.decrypt_page( + static_cast(index + 1), + encrypted.data() + (index * page_size), + image.data() + (index * page_size)); + } + + // Page one opens with the salt, where a plain database has its magic. + std::memcpy(image.data(), sqlite_file_magic, sqlite_file_magic_length); + + const auto log = read_file(database_path + "-wal"); + if (log.size() >= wal_header_length) + { + const auto magic = load_be32(log.data()); + const auto big_endian_checksums = magic == wal_magic_big_endian; + if (magic != wal_magic_little_endian && !big_endian_checksums) + throw sqlcipher_error{ + "`" + database_path + + "-wal` does not begin like a write-ahead log"}; + + const auto log_page_size = load_be32(log.data() + 8); + if (log_page_size != page_size) + throw sqlcipher_error{ + "the write-ahead log of `" + database_path + + "` uses a different page size from the database"}; + + const auto salt_1 = load_be32(log.data() + 16); + const auto salt_2 = load_be32(log.data() + 20); + + wal_checksum running; + running.accumulate(log.data(), 24, big_endian_checksums); + if (!running.matches(log.data() + 24)) + throw sqlcipher_error{ + "the write-ahead log of `" + database_path + + "` has a damaged header"}; + + // Walk the frames, keeping only those up to the last one that + // committed: anything after it belongs to a transaction that never + // finished, exactly as SQLite's own recovery decides. + const auto frame_length = wal_frame_header_length + page_size; + std::vector> frames; + size_t committed_frames = 0; + uint32_t committed_page_count = 0; + + for (size_t offset = wal_header_length; + offset + frame_length <= log.size(); offset += frame_length) + { + const auto* frame = log.data() + offset; + const auto page_number = load_be32(frame); + const auto truncate_to = load_be32(frame + 4); + + // A frame written after the log was reset carries the salt of the + // previous incarnation, and is not part of it. + if (load_be32(frame + 8) != salt_1 || + load_be32(frame + 12) != salt_2) + break; + + auto candidate = running; + candidate.accumulate(frame, 8, big_endian_checksums); + candidate.accumulate( + frame + wal_frame_header_length, page_size, + big_endian_checksums); + if (!candidate.matches(frame + 16)) + break; + + running = candidate; + frames.emplace_back(page_number, offset + wal_frame_header_length); + + if (truncate_to != 0) + { + committed_frames = frames.size(); + committed_page_count = truncate_to; + } + } + + if (committed_page_count != 0) + { + const auto committed_bytes = + static_cast(committed_page_count) * page_size; + image.resize(committed_bytes, 0); + } + + for (size_t index = 0; index < committed_frames; ++index) + { + const auto [page_number, payload_offset] = frames[index]; + const auto page_offset = + static_cast(page_number - 1) * page_size; + if (page_number == 0 || page_offset + page_size > image.size()) + continue; + + codec.decrypt_page( + page_number, log.data() + payload_offset, + image.data() + page_offset); + + if (page_number == 1) + std::memcpy( + image.data(), sqlite_file_magic, sqlite_file_magic_length); + } + } + + // The image no longer has a log, so mark it as using a rollback journal. + // Left as it is, SQLite would look for the log that has just been folded + // in, and refuse to open the database read-only without it. + if (image.size() > read_version_offset) + { + image[write_version_offset] = rollback_journal_version; + image[read_version_offset] = rollback_journal_version; + } + + return image; +} + +} // namespace djinterop::util::crypto diff --git a/src/djinterop/util/crypto/sqlcipher_wal.hpp b/src/djinterop/util/crypto/sqlcipher_wal.hpp new file mode 100644 index 0000000..db6a5ab --- /dev/null +++ b/src/djinterop/util/crypto/sqlcipher_wal.hpp @@ -0,0 +1,46 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include +#include +#include +#include + +#include "sqlcipher_codec.hpp" + +namespace djinterop::util::crypto +{ +/// Decrypt a SQLCipher database, merging its write-ahead log, into a plain +/// SQLite image. +/// +/// rekordbox leaves most of a fresh export in the log rather than in the +/// database file, so a reader that ignores it reports a nearly empty library +/// with no error at all. +/// +/// The returned image is a standard, unencrypted SQLite file, marked as using +/// a rollback journal: everything the log held has already been folded in. +/// +/// \param database_path Path of the encrypted database. +/// \param codec Codec built for the database, as by `make_codec_for`. +/// \return Returns a plain SQLite image. +/// \throws sqlcipher_error If the database cannot be read or decrypted. +std::vector decrypt_database_to_image( + const std::string& database_path, const sqlcipher_codec& codec); + +} // namespace djinterop::util::crypto diff --git a/src/djinterop/util/filesystem.cpp b/src/djinterop/util/filesystem.cpp index 03a61d3..b6a3e7d 100644 --- a/src/djinterop/util/filesystem.cpp +++ b/src/djinterop/util/filesystem.cpp @@ -44,6 +44,15 @@ bool path_exists(const std::string& directory) return (stat(directory.c_str(), &buf) == 0); } +bool path_is_directory(const std::string& path) +{ + struct stat buf; + if (stat(path.c_str(), &buf) != 0) + return false; + + return (buf.st_mode & S_IFMT) == S_IFDIR; +} + std::string get_filename(const std::string& file_path) { // TODO (haslersn): How to handle Windows path separator? diff --git a/src/djinterop/util/filesystem.hpp b/src/djinterop/util/filesystem.hpp index 5c72f8c..4e09216 100644 --- a/src/djinterop/util/filesystem.hpp +++ b/src/djinterop/util/filesystem.hpp @@ -24,6 +24,7 @@ namespace djinterop::util { void create_dir(const std::string& directory); bool path_exists(const std::string& directory); +bool path_is_directory(const std::string& path); std::string get_filename(const std::string& file_path); std::optional get_file_extension(const std::string& file_path); diff --git a/test/djinterop/onelibrary/content_table_test.cpp b/test/djinterop/onelibrary/content_table_test.cpp new file mode 100644 index 0000000..90ec9cd --- /dev/null +++ b/test/djinterop/onelibrary/content_table_test.cpp @@ -0,0 +1,247 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#define BOOST_TEST_MODULE onelibrary_content_table_test +#include + +#include +#include + +#include + +#include + +#include "../../../src/djinterop/onelibrary/content_table.hpp" +#include "../../../src/djinterop/onelibrary/onelibrary_context.hpp" +#include "../boost_test_printable.hpp" +#include "onelibrary_schema.hpp" + +namespace utf = boost::unit_test; +namespace ol = djinterop::onelibrary; + +namespace +{ +/// Build a context over an in-memory database holding a small library. +/// +/// The tables are read directly here, with no encryption in the way, which +/// keeps these tests clear of the deliberately expensive key derivation that +/// a real database needs. +std::shared_ptr make_context() +{ + sqlite::database db{":memory:"}; + for (const auto& statement : onelibrary_schema_statements()) + db << statement; + + db << "INSERT INTO artist VALUES (1, 'Aphex Twin', ''), " + "(2, 'Squarepusher', '')"; + db << "INSERT INTO album VALUES (1, 'Selected Ambient', 1, 0, 0, '')"; + db << "INSERT INTO genre VALUES (1, 'Electro')"; + db << "INSERT INTO label VALUES (1, 'Warp')"; + db << R"(INSERT INTO "key" VALUES (1, 'F#m'), (2, 'Camelot 8A'))"; + + // A fully populated track. + db << "INSERT INTO content (content_id, title, bpmx100, length, trackNo, " + "artist_id_artist, artist_id_composer, album_id, genre_id, " + "label_id, key_id, djComment, rating, releaseYear, path, fileName, " + "fileSize, bitrate, samplingRate) VALUES (1, 'Alpha Track', 12400, " + "391, 7, 1, 2, 1, 1, 1, 1, 'feelin good', 4, 2025, " + "'/Contents/Aphex/alpha.mp3', 'alpha.mp3', 6580703, 320, 44100)"; + + // Metadata that is present but empty, and a key notation not understood. + db << "INSERT INTO content (content_id, title, djComment, key_id, path) " + "VALUES (2, 'Beta Track', '', 2, '/Contents/Various/beta.flac')"; + + return std::make_shared("/device", std::move(db)); +} + +} // anonymous namespace + +BOOST_TEST_DECORATOR(*utf::description("get() resolves the lookup tables")) +BOOST_AUTO_TEST_CASE(get__a_populated_row__resolves_its_lookups) +{ + // Arrange + const ol::content_table content{make_context()}; + + // Act + const auto row = content.get(1); + + // Assert + BOOST_REQUIRE(row); + BOOST_CHECK_EQUAL(row->title.value(), "Alpha Track"); + BOOST_CHECK_EQUAL(row->artist.value(), "Aphex Twin"); + BOOST_CHECK_EQUAL(row->composer.value(), "Squarepusher"); + BOOST_CHECK_EQUAL(row->album.value(), "Selected Ambient"); + BOOST_CHECK_EQUAL(row->genre.value(), "Electro"); + BOOST_CHECK_EQUAL(row->label.value(), "Warp"); + BOOST_CHECK_EQUAL(row->key.value(), "F#m"); + BOOST_CHECK_EQUAL(row->bpm_x100.value(), 12400); + BOOST_CHECK_EQUAL(row->length_seconds.value(), 391); + BOOST_CHECK_EQUAL(row->rating_stars.value(), 4); + BOOST_CHECK_EQUAL(row->path.value(), "/Contents/Aphex/alpha.mp3"); +} + +BOOST_TEST_DECORATOR(*utf::description("get() reads an empty column as absent")) +BOOST_AUTO_TEST_CASE(get__an_empty_column__reads_as_absent) +{ + // Arrange + const ol::content_table content{make_context()}; + + // Act + const auto row = content.get(2); + + // Assert + BOOST_REQUIRE(row); + BOOST_CHECK(!row->comment); + BOOST_CHECK(!row->artist); + BOOST_CHECK(!row->bpm_x100); +} + +BOOST_TEST_DECORATOR(*utf::description("get() for a row that is not there")) +BOOST_AUTO_TEST_CASE(get__an_unknown_row__is_absent) +{ + // Arrange + const ol::content_table content{make_context()}; + + // Act + const auto row = content.get(404); + + // Assert + BOOST_CHECK(!row); +} + +BOOST_TEST_DECORATOR(*utf::description("all_ids() is ordered by identifier")) +BOOST_AUTO_TEST_CASE(all_ids__a_populated_table__is_ordered) +{ + // Arrange + const ol::content_table content{make_context()}; + + // Act + const auto ids = content.all_ids(); + + // Assert + BOOST_REQUIRE_EQUAL(ids.size(), 2u); + BOOST_CHECK_EQUAL(ids[0], 1); + BOOST_CHECK_EQUAL(ids[1], 2); +} + +BOOST_TEST_DECORATOR( + *utf::description("ids_by_path() matches with or without a separator")) +BOOST_AUTO_TEST_CASE(ids_by_path__either_spelling__finds_the_row) +{ + // Arrange + const ol::content_table content{make_context()}; + + // Act + const auto absolute = content.ids_by_path("/Contents/Aphex/alpha.mp3"); + const auto relative = content.ids_by_path("Contents/Aphex/alpha.mp3"); + + // Assert + BOOST_REQUIRE_EQUAL(absolute.size(), 1u); + BOOST_CHECK_EQUAL(absolute[0], 1); + BOOST_CHECK(relative == absolute); +} + +BOOST_TEST_DECORATOR(*utf::description("exists() for present and absent rows")) +BOOST_AUTO_TEST_CASE(exists__present_and_absent_rows__reports_each) +{ + // Arrange + const ol::content_table content{make_context()}; + + // Act / Assert + BOOST_CHECK(content.exists(1)); + BOOST_CHECK(!content.exists(404)); +} + +BOOST_TEST_DECORATOR( + *utf::description("to_snapshot() converts the units of the format")) +BOOST_AUTO_TEST_CASE(to_snapshot__a_populated_row__converts_its_units) +{ + // Arrange + const ol::content_table content{make_context()}; + const auto row = content.get(1); + BOOST_REQUIRE(row); + + // Act + const auto snapshot = ol::to_snapshot(*row); + + // Assert + BOOST_CHECK_CLOSE(snapshot.bpm.value(), 124.0, 0.001); + BOOST_CHECK_EQUAL(snapshot.duration.value().count(), 391000); + + // Ratings are whole stars in the database, and out of one hundred here. + BOOST_CHECK_EQUAL(snapshot.rating.value(), 80); + + // Paths are absolute within the device, and relative to it here. + BOOST_CHECK_EQUAL( + snapshot.relative_path.value(), "Contents/Aphex/alpha.mp3"); + + // No sample count is recorded, so it follows from the duration and rate. + BOOST_CHECK_EQUAL(snapshot.sample_count.value(), 391 * 44100); + BOOST_CHECK_EQUAL(snapshot.publisher.value(), "Warp"); + BOOST_CHECK(snapshot.key == djinterop::musical_key::f_sharp_minor); +} + +BOOST_TEST_DECORATOR( + *utf::description("to_snapshot() for data the database does not hold")) +BOOST_AUTO_TEST_CASE(to_snapshot__any_row__has_no_performance_data) +{ + // Arrange + const ol::content_table content{make_context()}; + const auto row = content.get(1); + BOOST_REQUIRE(row); + + // Act + const auto snapshot = ol::to_snapshot(*row); + + // Assert: rekordbox leaves these in the ANLZ files beside the database. + BOOST_CHECK(snapshot.beatgrid.empty()); + BOOST_CHECK(snapshot.waveform.empty()); + BOOST_CHECK(snapshot.hot_cues.empty()); + BOOST_CHECK(snapshot.loops.empty()); +} + +BOOST_TEST_DECORATOR( + *utf::description("parse_musical_key() for the notations rekordbox uses")) +BOOST_AUTO_TEST_CASE(parse_musical_key__known_notations__are_understood) +{ + // Act / Assert + BOOST_CHECK(ol::parse_musical_key("C") == djinterop::musical_key::c_major); + BOOST_CHECK(ol::parse_musical_key("Am") == djinterop::musical_key::a_minor); + BOOST_CHECK( + ol::parse_musical_key("F#m") == djinterop::musical_key::f_sharp_minor); + BOOST_CHECK( + ol::parse_musical_key("Bb") == djinterop::musical_key::b_flat_major); + + // The typographic accidentals mean the same as the ASCII ones. + BOOST_CHECK( + ol::parse_musical_key("F♯m") == djinterop::musical_key::f_sharp_minor); + BOOST_CHECK( + ol::parse_musical_key("B♭") == djinterop::musical_key::b_flat_major); +} + +BOOST_TEST_DECORATOR( + *utf::description("parse_musical_key() for notations it does not know")) +BOOST_AUTO_TEST_CASE(parse_musical_key__unknown_notations__are_not_guessed) +{ + // Act / Assert + BOOST_CHECK(!ol::parse_musical_key("")); + BOOST_CHECK(!ol::parse_musical_key("H")); + + // The Camelot and Open Key wheels are not read. + BOOST_CHECK(!ol::parse_musical_key("8A")); + BOOST_CHECK(!ol::parse_musical_key("Camelot 8A")); +} diff --git a/test/djinterop/onelibrary/database_test.cpp b/test/djinterop/onelibrary/database_test.cpp new file mode 100644 index 0000000..84b2bbf --- /dev/null +++ b/test/djinterop/onelibrary/database_test.cpp @@ -0,0 +1,852 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#define BOOST_TEST_MODULE onelibrary_database_test +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include "../../../src/djinterop/util/filesystem.hpp" +#include "../boost_test_printable.hpp" +#include "../sqlcipher_encryptor.hpp" +#include "../temporary_directory.hpp" +#include "onelibrary_schema.hpp" + +namespace utf = boost::unit_test; +namespace ol = djinterop::onelibrary; +namespace crypto = djinterop::util::crypto; + +namespace +{ +const std::string passphrase = "a passphrase for the fixture"; + +/// The encryptor that every fixture is written with. +/// +/// Key derivation is deliberately expensive -- the format stretches the +/// passphrase 256,000 times -- so one salt, and hence one derived key, is +/// shared by every fixture rather than made afresh for each. A database +/// carries its own salt, so nothing about reading one depends on this. +const sqlcipher_encryptor& fixture_encryptor() +{ + static const sqlcipher_encryptor encryptor{passphrase}; + return encryptor; +} + +/// Encrypt a plain SQLite file in place, as SQLCipher would have written it. +/// +/// The fixture is built as an ordinary database and encrypted afterwards, +/// which is the only direction the library offers: it decrypts a device to +/// read it, and never writes one. +void encrypt_in_place(const std::string& path) +{ + std::vector plain; + { + std::ifstream file{path, std::ios::binary}; + plain.assign( + std::istreambuf_iterator{file}, + std::istreambuf_iterator{}); + } + + const crypto::sqlcipher_parameters params; + BOOST_REQUIRE(!plain.empty()); + BOOST_REQUIRE_EQUAL(plain.size() % params.page_size, 0u); + + const auto& encryptor = fixture_encryptor(); + + std::vector encrypted(plain.size()); + for (size_t index = 0; index < plain.size() / params.page_size; ++index) + { + encryptor.encrypt_page( + static_cast(index + 1), + plain.data() + (index * params.page_size), + encrypted.data() + (index * params.page_size)); + } + + std::ofstream{path, std::ios::binary | std::ios::trunc}.write( + reinterpret_cast(encrypted.data()), + static_cast(encrypted.size())); +} + +/// Prepare a plain database to hold pages of the shape SQLCipher expects. +void prepare_plain_database(sqlite3* db) +{ + const crypto::sqlcipher_parameters params; + char* error = nullptr; + const auto sql = "PRAGMA page_size = " + std::to_string(params.page_size); + BOOST_REQUIRE_EQUAL( + sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &error), SQLITE_OK); + sqlite3_free(error); + + auto reserve = static_cast(params.reserve); + BOOST_REQUIRE_EQUAL( + sqlite3_file_control(db, "main", SQLITE_FCNTL_RESERVE_BYTES, &reserve), + SQLITE_OK); +} + +void execute(sqlite3* db, const std::string& sql) +{ + char* error = nullptr; + const auto rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &error); + const std::string message = error != nullptr ? error : ""; + sqlite3_free(error); + BOOST_REQUIRE_MESSAGE( + rc == SQLITE_OK, "failed to run \"" << sql << "\": " << message); +} + +void create_onelibrary_schema(sqlite3* db) +{ + for (const auto& statement : onelibrary_schema_statements()) + execute(db, statement); +} + +/// Build a device holding a small OneLibrary export. +/// +/// The schema is the one a real export carries, abbreviated to the tables this +/// library reads. The data covers the awkward cases: metadata that is absent, +/// present but empty, and non-ASCII; a key notation that is not understood; +/// and a nested playlist. +std::string make_device(const temporary_directory& temp_dir) +{ + const auto root = temp_dir.temp_dir; + djinterop::util::create_dir(root + "/PIONEER"); + djinterop::util::create_dir(root + "/PIONEER/rekordbox"); + + const auto path = root + "/" + ol::database_relative_path; + + sqlite3* db = nullptr; + BOOST_REQUIRE_EQUAL( + sqlite3_open_v2( + path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + nullptr), + SQLITE_OK); + + prepare_plain_database(db); + + create_onelibrary_schema(db); + + execute( + db, + "INSERT INTO artist VALUES (1, 'Aphex Twin', 'APHEX TWIN'), " + "(2, 'Кто-то', ''), (3, 'A Composer', '')"); + execute(db, "INSERT INTO album VALUES (1, 'Album One', 1, 0, 0, '')"); + execute(db, "INSERT INTO genre VALUES (1, 'Electro')"); + execute(db, "INSERT INTO label VALUES (1, 'Warp')"); + execute( + db, + "INSERT INTO \"key\" VALUES (1, 'F#m'), (2, 'Bb'), " + "(3, 'Camelot 8A')"); + + // A fully populated track. + execute( + db, + "INSERT INTO content (content_id, title, bpmx100, length, trackNo, " + "artist_id_artist, artist_id_composer, album_id, genre_id, label_id, " + "key_id, djComment, rating, releaseYear, path, fileName, fileSize, " + "bitrate, samplingRate, masterDbId, analysisDataFilePath, color_id) " + "VALUES (1, 'Alpha Track', 12400, " + "391, 7, 1, 3, 1, 1, 1, 1, 'feelin good', 4, 2025, " + "'/Contents/Aphex/alpha.mp3', 'alpha.mp3', 6580703, 320, 44100, " + "4056018032, '/PIONEER/USBANLZ/P016/0000875e/ANLZ0000.DAT', 6)"); + + // Non-ASCII metadata, an empty comment, and no rating. + execute( + db, + "INSERT INTO content (content_id, title, bpmx100, length, " + "artist_id_artist, key_id, djComment, rating, path, samplingRate, " + "masterDbId, color_id) VALUES (2, 'Бета Трек', 12800, 245, 2, 2, '', " + "0, '/Contents/Various/beta.flac', 48000, 4056018032, 0)"); + + // Almost nothing set, and a key notation this library does not know. + execute( + db, + "INSERT INTO content (content_id, title, key_id, rating, path) " + "VALUES (3, 'Gamma', 3, 5, '/Contents/Third/gamma.wav')"); + + execute( + db, + "INSERT INTO playlist VALUES (1, 1, 'Sets', 0, 0, 0), " + "(2, 1, 'Warm Up', 0, 0, 1), (3, 2, 'Peak Time', 0, 0, 0)"); + execute( + db, + "INSERT INTO playlist_content VALUES (2, 1, 1), (2, 3, 2), " + "(3, 2, 1)"); + execute( + db, + "INSERT INTO property VALUES ('FIXTURE', '1000', 3, " + "'2026-01-01', 0, 0)"); + + sqlite3_close(db); + encrypt_in_place(path); + return root; +} + +/// The device that the tests read, built once. +/// +/// Key derivation is deliberately expensive -- the format stretches the +/// passphrase 256,000 times -- so a test that built its own device and opened +/// it would pay for that twice over, every time. +struct shared_device +{ + temporary_directory temp_dir; + std::string path; + + shared_device() : path{make_device(temp_dir)} {} +}; + +const shared_device& device_fixture() +{ + static const shared_device instance; + return instance; +} + +/// The shared device, opened once. +djinterop::database loaded_database() +{ + static const djinterop::database db = + ol::load_database(device_fixture().path, passphrase); + return db; +} + +/// The shared device, opened once as a library. +const ol::library& loaded_library() +{ + static const ol::library lib{device_fixture().path, passphrase}; + return lib; +} + +} // anonymous namespace + +BOOST_TEST_DECORATOR(*utf::description( + "database_exists() finds a device by its root or by its file")) +BOOST_AUTO_TEST_CASE(database_exists__a_device__is_found) +{ + const auto& device = device_fixture().path; + + BOOST_CHECK(ol::database_exists(device)); + BOOST_CHECK(ol::database_exists(device + "/" + ol::database_relative_path)); + BOOST_CHECK(!ol::database_exists(device + "/nowhere")); +} + +BOOST_TEST_DECORATOR( + *utf::description("load_database() reports the identity of a device")) +BOOST_AUTO_TEST_CASE(load_database__a_device__reports_its_identity) +{ + const auto& device = device_fixture().path; + auto db = loaded_database(); + + BOOST_CHECK_EQUAL(db.version_name(), "OneLibrary 1000"); + BOOST_CHECK_EQUAL(db.uuid(), "4056018032"); + BOOST_CHECK_EQUAL(db.directory(), device); + + // The format keeps one tree, used as both playlists and crates. + BOOST_CHECK(!db.supports_feature( + djinterop::feature::playlists_and_crates_are_distinct)); + BOOST_CHECK( + db.supports_feature(djinterop::feature::supports_nested_playlists)); +} + +BOOST_TEST_DECORATOR( + *utf::description("load_database() accepts the database file itself")) +BOOST_AUTO_TEST_CASE(load_database__the_database_file_itself__is_accepted) +{ + const auto& device = device_fixture().path; + auto db = ol::load_database( + device + "/" + ol::database_relative_path, passphrase); + + BOOST_CHECK_EQUAL(db.tracks().size(), 3u); + + // Naming the file still identifies the device it belongs to, because + // track paths are relative to that and not to the file. + BOOST_CHECK_EQUAL(db.directory(), device); +} + +BOOST_TEST_DECORATOR( + *utf::description("load_database() refuses a wrong passphrase")) +BOOST_AUTO_TEST_CASE(load_database__a_wrong_passphrase__is_refused) +{ + const auto& device = device_fixture().path; + + BOOST_CHECK_THROW( + ol::load_database(device, "some other passphrase"), + djinterop::unsupported_database); +} + +BOOST_TEST_DECORATOR( + *utf::description("load_database() throws when no database is present")) +BOOST_AUTO_TEST_CASE(load_database__no_database__throws) +{ + temporary_directory temp_dir; + + BOOST_CHECK_THROW( + ol::load_database(temp_dir.temp_dir, passphrase), + djinterop::database_not_found); +} + +BOOST_TEST_DECORATOR( + *utf::description("tracks() reads every field of a populated track")) +BOOST_AUTO_TEST_CASE(tracks__a_populated_track__reads_every_field) +{ + auto db = loaded_database(); + + const auto track = db.track_by_id(1); + BOOST_REQUIRE(track); + + BOOST_CHECK_EQUAL(track->title().value(), "Alpha Track"); + BOOST_CHECK_EQUAL(track->artist().value(), "Aphex Twin"); + BOOST_CHECK_EQUAL(track->composer().value(), "A Composer"); + BOOST_CHECK_EQUAL(track->album().value(), "Album One"); + BOOST_CHECK_EQUAL(track->genre().value(), "Electro"); + BOOST_CHECK_EQUAL(track->publisher().value(), "Warp"); + BOOST_CHECK_EQUAL(track->comment().value(), "feelin good"); + + // Tempo is stored in hundredths of a beat per minute. + BOOST_CHECK_CLOSE(track->bpm().value(), 124.0, 0.001); + + // Duration is stored in whole seconds. + BOOST_CHECK_EQUAL(track->duration().value().count(), 391000); + + BOOST_CHECK_EQUAL(track->track_number().value(), 7); + BOOST_CHECK_EQUAL(track->year().value(), 2025); + BOOST_CHECK_EQUAL(track->bitrate().value(), 320); + BOOST_CHECK_CLOSE(track->sample_rate().value(), 44100.0, 0.001); + + // Four stars of five, on djinterop's scale of one hundred. + BOOST_CHECK_EQUAL(track->rating().value(), 80); + + BOOST_CHECK(track->key().value() == djinterop::musical_key::f_sharp_minor); + + // Paths are absolute within the device; djinterop wants them relative to + // its root. + BOOST_CHECK_EQUAL(track->relative_path(), "Contents/Aphex/alpha.mp3"); + BOOST_CHECK_EQUAL(track->filename(), "alpha.mp3"); + BOOST_CHECK_EQUAL(track->file_extension(), "mp3"); +} + +BOOST_TEST_DECORATOR( + *utf::description("tracks() reads sparse metadata as absent")) +BOOST_AUTO_TEST_CASE(tracks__sparse_metadata__reads_as_absent) +{ + auto db = loaded_database(); + + const auto track = db.track_by_id(3); + BOOST_REQUIRE(track); + + BOOST_CHECK_EQUAL(track->title().value(), "Gamma"); + BOOST_CHECK(!track->artist()); + BOOST_CHECK(!track->album()); + BOOST_CHECK(!track->bpm()); + BOOST_CHECK(!track->duration()); + BOOST_CHECK(!track->bitrate()); + BOOST_CHECK(!track->sample_rate()); + BOOST_CHECK(!track->sample_count()); + + // A key notation this library does not know reads as no key, rather than + // as a guess. + BOOST_CHECK(!track->key()); + + // An empty comment is metadata the track does not carry. + const auto other = db.track_by_id(2); + BOOST_REQUIRE(other); + BOOST_CHECK(!other->comment()); + BOOST_CHECK_EQUAL(other->title().value(), "Бета Трек"); + BOOST_CHECK_EQUAL(other->artist().value(), "Кто-то"); + BOOST_CHECK(other->key().value() == djinterop::musical_key::b_flat_major); +} + +BOOST_TEST_DECORATOR(*utf::description( + "tracks() implies a sample count from the duration and rate")) +BOOST_AUTO_TEST_CASE(tracks__a_known_duration_and_rate__implies_a_sample_count) +{ + auto db = loaded_database(); + + const auto track = db.track_by_id(1); + BOOST_REQUIRE(track); + + // The database records whole seconds and no sample count, so the count is + // only recoverable to that precision. + BOOST_CHECK_EQUAL(track->sample_count().value(), 391ull * 44100); +} + +BOOST_TEST_DECORATOR( + *utf::description("tracks_by_relative_path() finds a track by its path")) +BOOST_AUTO_TEST_CASE(tracks_by_relative_path__a_known_path__finds_the_track) +{ + auto db = loaded_database(); + + const auto found = db.tracks_by_relative_path("Contents/Aphex/alpha.mp3"); + BOOST_REQUIRE_EQUAL(found.size(), 1u); + BOOST_CHECK_EQUAL(found.front().id(), 1); + + // The same path written the way the database stores it. + BOOST_CHECK_EQUAL( + db.tracks_by_relative_path("/Contents/Aphex/alpha.mp3").size(), 1u); + + BOOST_CHECK(db.tracks_by_relative_path("nothing/here.mp3").empty()); +} + +BOOST_TEST_DECORATOR( + *utf::description("track_by_id() for a track that is not there")) +BOOST_AUTO_TEST_CASE(track_by_id__an_unknown_track__is_absent) +{ + auto db = loaded_database(); + + BOOST_CHECK(!db.track_by_id(999)); +} + +BOOST_TEST_DECORATOR(*utf::description("playlists walk a nested tree")) +BOOST_AUTO_TEST_CASE(playlists__a_nested_tree__is_walked) +{ + auto db = loaded_database(); + + const auto roots = db.root_playlists(); + BOOST_REQUIRE_EQUAL(roots.size(), 2u); + BOOST_CHECK_EQUAL(roots[0].name(), "Sets"); + BOOST_CHECK_EQUAL(roots[1].name(), "Peak Time"); + + // The folder itself holds no tracks; its child does. + BOOST_CHECK(roots[0].tracks().empty()); + + const auto children = roots[0].children(); + BOOST_REQUIRE_EQUAL(children.size(), 1u); + BOOST_CHECK_EQUAL(children[0].name(), "Warm Up"); + + const auto tracks = children[0].tracks(); + BOOST_REQUIRE_EQUAL(tracks.size(), 2u); + BOOST_CHECK_EQUAL(tracks[0].id(), 1); + BOOST_CHECK_EQUAL(tracks[1].id(), 3); + + BOOST_REQUIRE(children[0].parent()); + BOOST_CHECK_EQUAL(children[0].parent()->name(), "Sets"); + BOOST_CHECK(!roots[0].parent()); +} + +BOOST_TEST_DECORATOR(*utf::description("crates see the same tree as playlists")) +BOOST_AUTO_TEST_CASE(crates__the_same_tree_as_playlists__is_seen) +{ + auto db = loaded_database(); + + const auto roots = db.root_crates(); + BOOST_REQUIRE_EQUAL(roots.size(), 2u); + + const auto sets = db.root_crate_by_name("Sets"); + BOOST_REQUIRE(sets); + BOOST_CHECK_EQUAL(sets->descendants().size(), 1u); + + const auto warm_up = sets->sub_crate_by_name("Warm Up"); + BOOST_REQUIRE(warm_up); + BOOST_CHECK_EQUAL(warm_up->tracks().size(), 2u); + + BOOST_CHECK(!db.root_crate_by_name("No Such Crate")); + + const auto track = db.track_by_id(1); + BOOST_REQUIRE(track); + const auto containing = track->containing_crates(); + BOOST_REQUIRE_EQUAL(containing.size(), 1u); + BOOST_CHECK_EQUAL(containing.front().name(), "Warm Up"); +} + +BOOST_TEST_DECORATOR(*utf::description("every kind of write is refused")) +BOOST_AUTO_TEST_CASE(writes__every_kind__are_refused) +{ + auto db = loaded_database(); + + BOOST_CHECK_THROW( + db.create_root_crate("New"), djinterop::unsupported_operation); + BOOST_CHECK_THROW( + db.create_root_playlist("New"), djinterop::unsupported_operation); + BOOST_CHECK_THROW( + db.create_track(djinterop::track_snapshot{}), + djinterop::unsupported_operation); + + auto track = db.tracks().front(); + BOOST_CHECK_THROW( + track.set_title(std::string{"New"}), djinterop::unsupported_operation); + BOOST_CHECK_THROW(track.set_bpm(100.0), djinterop::unsupported_operation); + + auto crate = db.root_crates().front(); + BOOST_CHECK_THROW(crate.set_name("New"), djinterop::unsupported_operation); + BOOST_CHECK_THROW( + crate.create_sub_crate("New"), djinterop::unsupported_operation); + BOOST_CHECK_THROW(db.remove_crate(crate), djinterop::unsupported_operation); +} + +BOOST_TEST_DECORATOR(*utf::description("crates walk a deep tree breadth first")) +BOOST_AUTO_TEST_CASE( + crates__a_tree_several_levels_deep__is_walked_breadth_first) +{ + // The shared fixture is only one level deep, which does not exercise the + // recursion in `descendant_ids` or the order it returns. + temporary_directory temp_dir; + const auto path = temp_dir.temp_dir + "/deep.db"; + + sqlite3* db = nullptr; + BOOST_REQUIRE_EQUAL( + sqlite3_open_v2( + path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + nullptr), + SQLITE_OK); + prepare_plain_database(db); + create_onelibrary_schema(db); + + // Root + // +- Middle A (sequence 1) + // | +- Leaf A (sequence 1) + // +- Middle B (sequence 2) + execute( + db, + "INSERT INTO playlist VALUES (1, 1, 'Root', 0, 0, 0), " + "(2, 1, 'Middle A', 0, 0, 1), (3, 2, 'Middle B', 0, 0, 1), " + "(4, 1, 'Leaf A', 0, 0, 2)"); + execute( + db, + "INSERT INTO property VALUES ('FIXTURE', '1000', 0, " + "'2026-01-01', 0, 0)"); + sqlite3_close(db); + encrypt_in_place(path); + + auto loaded = ol::load_database(path, passphrase); + const auto root = loaded.root_crate_by_name("Root"); + BOOST_REQUIRE(root); + + const auto descendants = root->descendants(); + BOOST_REQUIRE_EQUAL(descendants.size(), 3u); + BOOST_CHECK_EQUAL(descendants[0].name(), "Middle A"); + BOOST_CHECK_EQUAL(descendants[1].name(), "Middle B"); + BOOST_CHECK_EQUAL(descendants[2].name(), "Leaf A"); + + // Children stay one level deep, unlike descendants. + BOOST_CHECK_EQUAL(root->children().size(), 2u); +} + +BOOST_TEST_DECORATOR(*utf::description( + "load_database() reads a database whose log was checkpointed")) +BOOST_AUTO_TEST_CASE(load_database__a_checkpointed_log__is_read) +{ + // A real export is written in WAL mode and checkpointed on eject, which + // removes the log but leaves the header declaring the database + // write-ahead-logged. SQLite refuses to open one of those read-only + // without the log, so the header has to be rewritten as it is decrypted. + temporary_directory temp_dir; + const auto path = temp_dir.temp_dir + "/checkpointed.db"; + + sqlite3* db = nullptr; + BOOST_REQUIRE_EQUAL( + sqlite3_open_v2( + path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + nullptr), + SQLITE_OK); + prepare_plain_database(db); + create_onelibrary_schema(db); + execute(db, "PRAGMA journal_mode = WAL"); + execute( + db, + "INSERT INTO content (content_id, title, path) " + "VALUES (1, 'Checkpointed', '/a.mp3')"); + execute( + db, + "INSERT INTO property VALUES ('FIXTURE', '1000', 1, " + "'2026-01-01', 0, 0)"); + + // Closing folds the log back in, exactly as ejecting a device does, + // leaving nothing beside the database but its header still saying WAL. + sqlite3_close(db); + std::remove((path + "-wal").c_str()); + + encrypt_in_place(path); + + auto loaded = ol::load_database(path, passphrase); + const auto tracks = loaded.tracks(); + BOOST_REQUIRE_EQUAL(tracks.size(), 1u); + BOOST_CHECK_EQUAL(tracks[0].title().value(), "Checkpointed"); +} + +BOOST_TEST_DECORATOR( + *utf::description("verify() rejects a database missing its tables")) +BOOST_AUTO_TEST_CASE(verify__a_database_missing_its_tables__is_rejected) +{ + temporary_directory temp_dir; + const auto path = temp_dir.temp_dir + "/not-onelibrary.db"; + + sqlite3* db = nullptr; + BOOST_REQUIRE_EQUAL( + sqlite3_open_v2( + path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + nullptr), + SQLITE_OK); + prepare_plain_database(db); + execute(db, "CREATE TABLE something_else(id INTEGER PRIMARY KEY)"); + sqlite3_close(db); + encrypt_in_place(path); + + BOOST_CHECK_THROW( + ol::load_database(path, passphrase), djinterop::database_inconsistency); +} + +namespace +{ +/// Read a whole file. +std::vector read_file(const std::string& path) +{ + std::ifstream file{path, std::ios::binary}; + return std::vector{ + std::istreambuf_iterator{file}, std::istreambuf_iterator{}}; +} + +void write_file(const std::string& path, const std::vector& data) +{ + std::ofstream{path, std::ios::binary | std::ios::trunc}.write( + reinterpret_cast(data.data()), + static_cast(data.size())); +} + +uint32_t load_be32(const uint8_t* p) +{ + return (static_cast(p[0]) << 24) | + (static_cast(p[1]) << 16) | + (static_cast(p[2]) << 8) | static_cast(p[3]); +} + +void store_be32(uint8_t* p, uint32_t value) +{ + p[0] = static_cast(value >> 24); + p[1] = static_cast(value >> 16); + p[2] = static_cast(value >> 8); + p[3] = static_cast(value); +} + +/// Encrypt a plain database and its write-ahead log the way SQLCipher does. +/// +/// SQLCipher encrypts the page inside each log frame, and SQLite checksums the +/// frame over the bytes as they end up in the file -- so over the ciphertext. +/// Reproducing that here is what makes this a test of reading a real log +/// rather than of reading one this library made up. +void encrypt_database_and_log( + const std::string& path, const sqlcipher_encryptor& encryptor) +{ + const auto& params = encryptor.params(); + + auto plain = read_file(path); + BOOST_REQUIRE_EQUAL(plain.size() % params.page_size, 0u); + std::vector encrypted(plain.size()); + for (size_t index = 0; index < plain.size() / params.page_size; ++index) + encryptor.encrypt_page( + static_cast(index + 1), + plain.data() + (index * params.page_size), + encrypted.data() + (index * params.page_size)); + write_file(path, encrypted); + + auto log = read_file(path + "-wal"); + BOOST_REQUIRE_GE(log.size(), 32u); + + // The checksums of the log are computed over its contents in the byte + // order its magic number selects, and stored big-endian. + const auto big_endian = (load_be32(log.data()) & 1) != 0; + const auto read_word = [&](const uint8_t* p) + { + if (big_endian) + return load_be32(p); + + return static_cast( + (static_cast(p[3]) << 24) | + (static_cast(p[2]) << 16) | + (static_cast(p[1]) << 8) | static_cast(p[0])); + }; + + uint32_t s0 = 0; + uint32_t s1 = 0; + const auto accumulate = [&](const uint8_t* data, size_t length) + { + for (size_t offset = 0; offset + 8 <= length; offset += 8) + { + s0 += read_word(data + offset) + s1; + s1 += read_word(data + offset + 4) + s0; + } + }; + + accumulate(log.data(), 24); + BOOST_REQUIRE_EQUAL(s0, load_be32(log.data() + 24)); + BOOST_REQUIRE_EQUAL(s1, load_be32(log.data() + 28)); + + const auto frame_length = 24 + params.page_size; + size_t frames = 0; + for (size_t offset = 32; offset + frame_length <= log.size(); + offset += frame_length) + { + auto* frame = log.data() + offset; + const auto page_number = load_be32(frame); + + std::vector page(params.page_size); + encryptor.encrypt_page(page_number, frame + 24, page.data()); + std::memcpy(frame + 24, page.data(), page.size()); + + accumulate(frame, 8); + accumulate(frame + 24, params.page_size); + store_be32(frame + 16, s0); + store_be32(frame + 20, s1); + ++frames; + } + + BOOST_REQUIRE_GT(frames, 0u); + write_file(path + "-wal", log); +} + +} // anonymous namespace + +BOOST_TEST_DECORATOR( + *utf::description("load_database() reads data left in the write-ahead log")) +BOOST_AUTO_TEST_CASE(load_database__data_left_in_the_log__is_read) +{ + // rekordbox leaves most of a fresh export in the log rather than in the + // database file, so a reader that ignores it reports a nearly empty + // library, with no error at all. + temporary_directory temp_dir; + const auto device = temp_dir.temp_dir + "/device"; + djinterop::util::create_dir(device); + djinterop::util::create_dir(device + "/PIONEER"); + djinterop::util::create_dir(device + "/PIONEER/rekordbox"); + + const auto path = device + "/" + std::string{ol::database_relative_path}; + + sqlite3* db = nullptr; + BOOST_REQUIRE_EQUAL( + sqlite3_open_v2( + path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + nullptr), + SQLITE_OK); + prepare_plain_database(db); + create_onelibrary_schema(db); + execute( + db, + "INSERT INTO content (content_id, title, path) " + "VALUES (1, 'In The Database', '/a.mp3')"); + execute( + db, + "INSERT INTO property VALUES ('FIXTURE', '1000', 2, " + "'2026-01-01', 0, 0)"); + + // From here on, everything lands in the log and stays there. + execute(db, "PRAGMA journal_mode = WAL"); + execute(db, "PRAGMA wal_autocheckpoint = 0"); + execute( + db, + "INSERT INTO content (content_id, title, path) " + "VALUES (2, 'In The Log', '/b.mp3')"); + + // Copy the pair aside while the connection is open, because closing it + // would fold the log back in. + const auto db_copy = temp_dir.temp_dir + "/copy.db"; + write_file(db_copy, read_file(path)); + write_file(db_copy + "-wal", read_file(path + "-wal")); + sqlite3_close(db); + + write_file(path, read_file(db_copy)); + write_file(path + "-wal", read_file(db_copy + "-wal")); + + encrypt_database_and_log(path, fixture_encryptor()); + + auto loaded = ol::load_database(device, passphrase); + const auto tracks = loaded.tracks(); + BOOST_REQUIRE_EQUAL(tracks.size(), 2u); + BOOST_CHECK_EQUAL(tracks[0].title().value(), "In The Database"); + BOOST_CHECK_EQUAL(tracks[1].title().value(), "In The Log"); +} + +BOOST_TEST_DECORATOR( + *utf::description("library reaches the device through both interfaces")) +BOOST_AUTO_TEST_CASE(library__a_device__is_read_through_its_database) +{ + // Arrange + const auto& lib = loaded_library(); + + // Act + auto db = lib.db(); + + // Assert + BOOST_CHECK_EQUAL(lib.directory(), device_fixture().path); + BOOST_CHECK_EQUAL(db.directory(), device_fixture().path); + BOOST_CHECK_EQUAL(db.tracks().size(), 3u); +} + +BOOST_TEST_DECORATOR(*utf::description( + "library::analysis_path() gives a path relative to the device")) +BOOST_AUTO_TEST_CASE(analysis_path__a_track_with_analysis_data__is_relative) +{ + // Arrange + const auto& lib = loaded_library(); + + // Act + const auto path = lib.analysis_path(1); + + // Assert + BOOST_REQUIRE(path); + BOOST_CHECK_EQUAL(*path, "PIONEER/USBANLZ/P016/0000875e/ANLZ0000.DAT"); +} + +BOOST_TEST_DECORATOR(*utf::description( + "library::analysis_path() for a track that carries none, and for one that " + "is not there")) +BOOST_AUTO_TEST_CASE(analysis_path__no_analysis_data__is_absent) +{ + // Arrange + const auto& lib = loaded_library(); + + // Act, Assert + BOOST_CHECK(!lib.analysis_path(2)); + BOOST_CHECK(!lib.analysis_path(1234)); +} + +BOOST_TEST_DECORATOR(*utf::description( + "library::key_name() gives back the notation the device holds")) +BOOST_AUTO_TEST_CASE(key_name__any_track__is_the_notation_on_the_device) +{ + // Arrange + const auto& lib = loaded_library(); + + // Act, Assert + BOOST_CHECK_EQUAL(lib.key_name(1).value(), "F#m"); + BOOST_CHECK_EQUAL(lib.key_name(2).value(), "Bb"); + + // A notation the library does not parse is still given back whole. + BOOST_CHECK_EQUAL(lib.key_name(3).value(), "Camelot 8A"); + + BOOST_CHECK(!lib.key_name(1234)); +} + +BOOST_TEST_DECORATOR( + *utf::description("library::color_id() reads the colour of a track")) +BOOST_AUTO_TEST_CASE(color_id__marked_and_unmarked_tracks__reports_each) +{ + // Arrange + const auto& lib = loaded_library(); + + // Act, Assert + BOOST_CHECK_EQUAL(lib.color_id(1).value(), 6); + + // A colour of zero is no colour, as is a column that is not set at all. + BOOST_CHECK(!lib.color_id(2)); + BOOST_CHECK(!lib.color_id(3)); + BOOST_CHECK(!lib.color_id(1234)); +} diff --git a/test/djinterop/onelibrary/onelibrary_schema.hpp b/test/djinterop/onelibrary/onelibrary_schema.hpp new file mode 100644 index 0000000..f2bb1e3 --- /dev/null +++ b/test/djinterop/onelibrary/onelibrary_schema.hpp @@ -0,0 +1,75 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include +#include + +/// The schema a real OneLibrary export carries, abbreviated to the tables that +/// this library reads. +/// +/// Nothing in a real export is declared `NOT NULL`, and no foreign key is +/// enforced, so the fixtures can leave any column unset. +/// +/// The published description of the format, which the pyrekordbox project +/// documents as Device Library Plus, lists neither `content.djPlayCount` nor +/// `album.image_id`, and gives `playlist_content` and `property` primary keys +/// that an export does not carry. What a device holds is what is written +/// here. +inline const std::vector& onelibrary_schema_statements() +{ + static const std::vector statements{ + "CREATE TABLE content(content_id integer primary key, title varchar, " + "titleForSearch varchar, subtitle varchar, bpmx100 integer, " + "length integer, trackNo integer, discNo integer, " + "artist_id_artist integer, artist_id_remixer integer, " + "artist_id_originalArtist integer, artist_id_composer integer, " + "artist_id_lyricist integer, album_id integer, genre_id integer, " + "label_id integer, key_id integer, color_id integer, " + "image_id integer, djComment varchar, rating integer, " + "releaseYear integer, releaseDate varchar, dateCreated varchar, " + "dateAdded varchar, path varchar, fileName varchar, " + "fileSize integer, fileType integer, bitrate integer, " + "bitDepth integer, samplingRate integer, isrc varchar, " + "djPlayCount integer, isHotCueAutoLoadOn integer, " + "isKuvoDeliverStatusOn integer, kuvoDeliveryComment varchar, " + "masterDbId integer, masterContentId integer, " + "analysisDataFilePath varchar, analysedBits integer, " + "contentLink integer, hasModified integer, cueUpdateCount integer, " + "analysisDataUpdateCount integer, informationUpdateCount integer)", + "CREATE TABLE artist(artist_id integer primary key, " + "name varchar, nameForSearch varchar)", + "CREATE TABLE album(album_id integer primary key, " + "name varchar, artist_id integer, image_id integer, " + "isComplation integer, nameForSearch varchar)", + "CREATE TABLE genre(genre_id integer primary key, name varchar)", + "CREATE TABLE label(label_id integer primary key, name varchar)", + "CREATE TABLE \"key\"(key_id integer primary key, name varchar)", + "CREATE TABLE playlist(playlist_id integer primary key, " + "sequenceNo integer, name varchar, image_id integer, " + "attribute integer, playlist_id_parent integer)", + "CREATE TABLE playlist_content(playlist_id integer, " + "content_id integer, sequenceNo integer)", + "CREATE TABLE property(deviceName varchar, " + "dbVersion varchar, numberOfContents integer, " + "createdDate varchar, backGroundColorType integer, " + "myTagMasterDBID integer)", + }; + + return statements; +} diff --git a/test/djinterop/onelibrary/playlist_table_test.cpp b/test/djinterop/onelibrary/playlist_table_test.cpp new file mode 100644 index 0000000..7b4f445 --- /dev/null +++ b/test/djinterop/onelibrary/playlist_table_test.cpp @@ -0,0 +1,208 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#define BOOST_TEST_MODULE onelibrary_playlist_table_test +#include + +#include +#include + +#include + +#include "../../../src/djinterop/onelibrary/onelibrary_context.hpp" +#include "../../../src/djinterop/onelibrary/playlist_table.hpp" +#include "../boost_test_printable.hpp" +#include "onelibrary_schema.hpp" + +namespace utf = boost::unit_test; +namespace ol = djinterop::onelibrary; + +namespace +{ +/// Build a context over an in-memory database holding a tree of playlists. +/// +/// Sets (no parent recorded) +/// +- Warm Up (sequence 1) +/// | +- Peak Time (sequence 1) +/// +- Cool Down (sequence 2) +/// Practice (parent of zero, which means the same as none) +/// +/// The two spellings of a root are both present, as rekordbox writes both. +std::shared_ptr make_context() +{ + sqlite::database db{":memory:"}; + for (const auto& statement : onelibrary_schema_statements()) + db << statement; + + db << "INSERT INTO playlist (playlist_id, sequenceNo, name, " + "playlist_id_parent) VALUES (1, 1, 'Sets', NULL), " + "(2, 1, 'Warm Up', 1), (3, 2, 'Cool Down', 1), " + "(4, 1, 'Peak Time', 2), (5, 2, 'Practice', 0)"; + + db << "INSERT INTO content (content_id, title) VALUES (10, 'Alpha'), " + "(11, 'Beta'), (12, 'Gamma')"; + + // The second track of `Warm Up` is the one that `Peak Time` also holds. + db << "INSERT INTO playlist_content VALUES (2, 11, 1), (2, 10, 2), " + "(4, 10, 1)"; + + return std::make_shared("/device", std::move(db)); +} + +} // anonymous namespace + +BOOST_TEST_DECORATOR(*utf::description("get() reads one row of the tree")) +BOOST_AUTO_TEST_CASE(get__a_child__reads_its_row) +{ + // Arrange + const ol::playlist_table playlists{make_context()}; + + // Act + const auto row = playlists.get(2); + + // Assert + BOOST_REQUIRE(row); + BOOST_CHECK_EQUAL(row->id, 2); + BOOST_CHECK_EQUAL(row->name, "Warm Up"); + BOOST_REQUIRE(row->parent_id); + BOOST_CHECK_EQUAL(*row->parent_id, 1); + BOOST_CHECK_EQUAL(row->sequence_number.value(), 1); +} + +BOOST_TEST_DECORATOR( + *utf::description("get() treats a parent of zero as no parent")) +BOOST_AUTO_TEST_CASE(get__a_parent_of_zero__reads_as_no_parent) +{ + // Arrange + const ol::playlist_table playlists{make_context()}; + + // Act + const auto row = playlists.get(5); + + // Assert + BOOST_REQUIRE(row); + BOOST_CHECK(!row->parent_id); +} + +BOOST_TEST_DECORATOR(*utf::description("get() for a row that is not there")) +BOOST_AUTO_TEST_CASE(get__an_unknown_playlist__is_absent) +{ + // Arrange + const ol::playlist_table playlists{make_context()}; + + // Act / Assert + BOOST_CHECK(!playlists.get(404)); + BOOST_CHECK(!playlists.exists(404)); + BOOST_CHECK(playlists.exists(1)); +} + +BOOST_TEST_DECORATOR( + *utf::description("root_ids() finds both spellings of a root")) +BOOST_AUTO_TEST_CASE(root_ids__both_spellings__are_roots) +{ + // Arrange + const ol::playlist_table playlists{make_context()}; + + // Act + const auto ids = playlists.root_ids(); + + // Assert + BOOST_REQUIRE_EQUAL(ids.size(), 2u); + BOOST_CHECK_EQUAL(ids[0], 1); + BOOST_CHECK_EQUAL(ids[1], 5); +} + +BOOST_TEST_DECORATOR(*utf::description("child_ids() is in sibling order")) +BOOST_AUTO_TEST_CASE(child_ids__several_siblings__are_in_sequence) +{ + // Arrange + const ol::playlist_table playlists{make_context()}; + + // Act + const auto ids = playlists.child_ids(1); + + // Assert + BOOST_REQUIRE_EQUAL(ids.size(), 2u); + BOOST_CHECK_EQUAL(ids[0], 2); + BOOST_CHECK_EQUAL(ids[1], 3); +} + +BOOST_TEST_DECORATOR(*utf::description("descendant_ids() is breadth first")) +BOOST_AUTO_TEST_CASE(descendant_ids__a_deep_tree__is_breadth_first) +{ + // Arrange + const ol::playlist_table playlists{make_context()}; + + // Act + const auto ids = playlists.descendant_ids(1); + + // Assert + BOOST_REQUIRE_EQUAL(ids.size(), 3u); + BOOST_CHECK_EQUAL(ids[0], 2); // Warm Up + BOOST_CHECK_EQUAL(ids[1], 3); // Cool Down + BOOST_CHECK_EQUAL(ids[2], 4); // Peak Time, one level deeper +} + +BOOST_TEST_DECORATOR( + *utf::description("find_root() and find_child() look up by name")) +BOOST_AUTO_TEST_CASE(find__a_known_name__is_found_at_its_own_level) +{ + // Arrange + const ol::playlist_table playlists{make_context()}; + + // Act / Assert + BOOST_CHECK(playlists.find_root("Sets") == 1); + BOOST_CHECK(playlists.find_child(1, "Cool Down") == 3); + + // A child is not a root, and a name from elsewhere is not a child. + BOOST_CHECK(!playlists.find_root("Warm Up")); + BOOST_CHECK(!playlists.find_child(1, "Peak Time")); +} + +BOOST_TEST_DECORATOR(*utf::description("track_ids() is in playlist order")) +BOOST_AUTO_TEST_CASE(track_ids__a_populated_playlist__is_in_order) +{ + // Arrange + const ol::playlist_table playlists{make_context()}; + + // Act + const auto ids = playlists.track_ids(2); + + // Assert: the sequence numbers, and not the order the rows were written. + BOOST_REQUIRE_EQUAL(ids.size(), 2u); + BOOST_CHECK_EQUAL(ids[0], 11); + BOOST_CHECK_EQUAL(ids[1], 10); +} + +BOOST_TEST_DECORATOR( + *utf::description("playlists_containing() for a track held twice")) +BOOST_AUTO_TEST_CASE(playlists_containing__a_shared_track__finds_each_holder) +{ + // Arrange + const ol::playlist_table playlists{make_context()}; + + // Act + const auto holders = playlists.playlists_containing(10); + + // Assert + BOOST_REQUIRE_EQUAL(holders.size(), 2u); + BOOST_CHECK_EQUAL(holders[0], 2); + BOOST_CHECK_EQUAL(holders[1], 4); + + // A track in no playlist at all is held by nothing. + BOOST_CHECK(playlists.playlists_containing(12).empty()); +} diff --git a/test/djinterop/sqlcipher_encryptor.hpp b/test/djinterop/sqlcipher_encryptor.hpp new file mode 100644 index 0000000..b54f7b2 --- /dev/null +++ b/test/djinterop/sqlcipher_encryptor.hpp @@ -0,0 +1,134 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "../../src/djinterop/util/crypto/aes.hpp" +#include "../../src/djinterop/util/crypto/sha512.hpp" +#include "../../src/djinterop/util/crypto/sqlcipher_codec.hpp" + +/// Writes SQLCipher 4 pages, so that a test can build a database for the +/// library to read. +/// +/// The library only ever decrypts, so the encrypting direction lives here +/// rather than beside it. Writing the pages from the format description +/// independently also makes a round trip a check of the format, rather than a +/// check that the library agrees with itself. +class sqlcipher_encryptor +{ +public: + using parameters = djinterop::util::crypto::sqlcipher_parameters; + using salt_type = djinterop::util::crypto::sqlcipher_salt; + + explicit sqlcipher_encryptor( + const std::string& passphrase, const parameters& params = {}) : + params_{params}, + salt_{random_bytes()}, + page_key_{djinterop::util::crypto::pbkdf2_hmac_sha512( + reinterpret_cast(passphrase.data()), + passphrase.size(), salt_.data(), salt_.size(), + params.kdf_iterations, djinterop::util::crypto::aes256_key_length)}, + hmac_key_{derive_hmac_key()}, cipher_{page_key_.data()} + { + } + + /// The parameters the pages are written under. + [[nodiscard]] const parameters& params() const noexcept { return params_; } + + /// The salt written into the first page, with which a codec that is to + /// read the database has to be built. + [[nodiscard]] const salt_type& salt() const noexcept { return salt_; } + + /// Encrypt one page, of `page_size` bytes, under a fresh random vector. + void encrypt_page( + uint32_t page_number, const uint8_t* decrypted, + uint8_t* encrypted) const + { + using namespace djinterop::util::crypto; + + // Page one carries the salt where a plain SQLite file has its magic, + // so its ciphertext starts after it. + const auto offset = page_number == 1 ? sqlcipher_salt_length : 0; + const auto length = params_.payload_size() - offset; + const auto iv = random_bytes>(); + + cipher_.encrypt( + iv.data(), decrypted + offset, encrypted + offset, length); + + if (page_number == 1) + std::memcpy(encrypted, salt_.data(), salt_.size()); + + // The tag covers ciphertext || IV || page number, the last as a + // little-endian 32-bit integer, which binds a page to its position. + std::vector message; + message.insert( + message.end(), encrypted + offset, encrypted + offset + length); + message.insert(message.end(), iv.begin(), iv.end()); + for (int shift : {0, 8, 16, 24}) + message.push_back(static_cast(page_number >> shift)); + + const auto tag = hmac_sha512( + hmac_key_.data(), hmac_key_.size(), message.data(), message.size()); + + auto* reserve = encrypted + params_.payload_size(); + std::memcpy(reserve, iv.data(), iv.size()); + std::memcpy(reserve + iv.size(), tag.data(), tag.size()); + + const auto used = iv.size() + tag.size(); + if (params_.reserve > used) + std::memset(reserve + used, 0, params_.reserve - used); + } + +private: + template static Bytes random_bytes() + { + static std::random_device rng; + static std::uniform_int_distribution byte_dist{0, 255}; + + Bytes bytes{}; + for (auto& byte : bytes) + byte = static_cast(byte_dist(rng)); + + return bytes; + } + + /// The tag key is derived from the page key, under the salt with every + /// byte flipped by 0x3a, as SQLCipher does it. + [[nodiscard]] std::vector derive_hmac_key() const + { + salt_type hmac_salt{}; + for (size_t index = 0; index < salt_.size(); ++index) + hmac_salt[index] = static_cast(salt_[index] ^ 0x3a); + + return djinterop::util::crypto::pbkdf2_hmac_sha512( + page_key_.data(), page_key_.size(), hmac_salt.data(), + hmac_salt.size(), params_.hmac_kdf_iterations, 32); + } + + parameters params_; + salt_type salt_; + std::vector page_key_; + std::vector hmac_key_; + djinterop::util::crypto::aes256_cbc cipher_; +}; diff --git a/test/djinterop/util/crypto_test.cpp b/test/djinterop/util/crypto_test.cpp new file mode 100644 index 0000000..4bedbdb --- /dev/null +++ b/test/djinterop/util/crypto_test.cpp @@ -0,0 +1,300 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#define BOOST_TEST_MODULE crypto_test +#include + +#include +#include +#include + +#include "../../../src/djinterop/util/crypto/aes.hpp" +#include "../../../src/djinterop/util/crypto/sha512.hpp" +#include "../../../src/djinterop/util/crypto/sqlcipher_codec.hpp" +#include "../sqlcipher_encryptor.hpp" + +namespace utf = boost::unit_test; + +using namespace djinterop::util::crypto; + +namespace +{ +std::string to_hex(const uint8_t* data, size_t length) +{ + static const char* digits = "0123456789abcdef"; + std::string out; + out.reserve(length * 2); + for (size_t i = 0; i < length; ++i) + { + out.push_back(digits[data[i] >> 4]); + out.push_back(digits[data[i] & 0x0f]); + } + return out; +} + +const uint8_t* bytes_of(const std::string& s) +{ + return reinterpret_cast(s.data()); +} + +/// Parameters with a cheap key derivation. +/// +/// The real format stretches the passphrase 256,000 times, which is the point +/// of it, but paying that in every test makes the suite slow to no purpose, +/// especially in an unoptimised build. Tests about the page format use these; +/// the tests about key derivation itself use the real ones. +sqlcipher_parameters cheap_parameters() +{ + sqlcipher_parameters params; + params.kdf_iterations = 1000; + return params; +} + +} // anonymous namespace + +BOOST_TEST_DECORATOR(*utf::description("sha512 matches the published vectors")) +BOOST_AUTO_TEST_CASE(sha512__known_vectors__match) +{ + // FIPS 180-4 test vectors. + const std::string abc = "abc"; + const auto digest = sha512::hash(bytes_of(abc), abc.size()); + BOOST_CHECK_EQUAL( + to_hex(digest.data(), digest.size()), + "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a" + "2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f"); + + const std::string empty; + const auto empty_digest = sha512::hash(bytes_of(empty), 0); + BOOST_CHECK_EQUAL( + to_hex(empty_digest.data(), empty_digest.size()), + "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce" + "47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e"); +} + +BOOST_TEST_DECORATOR( + *utf::description("sha512 streamed in chunks matches one shot")) +BOOST_AUTO_TEST_CASE(sha512__streamed_in_chunks__matches_one_shot) +{ + const std::string input(1000, 'x'); + const auto expected = sha512::hash(bytes_of(input), input.size()); + + // Chunk sizes that divide neither the block length nor each other, so the + // buffering path is exercised at every alignment. + sha512 streamed; + for (size_t offset = 0; offset < input.size(); offset += 7) + streamed.update( + bytes_of(input) + offset, + std::min(7, input.size() - offset)); + + BOOST_CHECK(streamed.finalise() == expected); +} + +BOOST_TEST_DECORATOR( + *utf::description("hmac_sha512 matches the RFC 4231 vector")) +BOOST_AUTO_TEST_CASE(hmac_sha512__rfc_4231_vector__matches) +{ + const std::vector key(20, 0x0b); + const std::string data = "Hi There"; + const auto tag = + hmac_sha512(key.data(), key.size(), bytes_of(data), data.size()); + + BOOST_CHECK_EQUAL( + to_hex(tag.data(), tag.size()), + "87aa7cdea5ef619d4ff0b4241a1d6cb02379f4e2ce4ec2787ad0b30545e17cde" + "daa833b7d6b8a702038b274eaea3f4e4be9d914eeb61f1702e696c203a126854"); +} + +BOOST_TEST_DECORATOR( + *utf::description("pbkdf2_hmac_sha512 matches the published vector")) +BOOST_AUTO_TEST_CASE(pbkdf2_hmac_sha512__known_vector__matches) +{ + const std::string password = "passwd"; + const std::string salt = "salt"; + const auto derived = pbkdf2_hmac_sha512( + bytes_of(password), password.size(), bytes_of(salt), salt.size(), 1, + 64); + + BOOST_CHECK_EQUAL( + to_hex(derived.data(), derived.size()), + "c74319d99499fc3e9013acff597c23c5baf0a0bec5634c46b8352b793e324723" + "d55caa76b2b25c43402dcfdc06cdcf66f95b7d0429420b39520006749c51a04e"); +} + +BOOST_TEST_DECORATOR(*utf::description( + "pbkdf2_hmac_sha512 spans several blocks for a long output")) +BOOST_AUTO_TEST_CASE(pbkdf2_hmac_sha512__long_output__spans_several_blocks) +{ + // More than one digest of output, so the block-index loop is exercised. + const std::string password = "passwd"; + const std::string salt = "salt"; + const auto derived = pbkdf2_hmac_sha512( + bytes_of(password), password.size(), bytes_of(salt), salt.size(), 1, + 100); + + BOOST_REQUIRE_EQUAL(derived.size(), 100u); + + // Its first 64 bytes are the same derivation as above. + const auto shorter = pbkdf2_hmac_sha512( + bytes_of(password), password.size(), bytes_of(salt), salt.size(), 1, + 64); + BOOST_CHECK( + std::vector(derived.begin(), derived.begin() + 64) == shorter); +} + +BOOST_TEST_DECORATOR(*utf::description("aes256 matches the FIPS 197 vector")) +BOOST_AUTO_TEST_CASE(aes256__fips_197_vector__matches) +{ + uint8_t key[aes256_key_length]; + for (size_t i = 0; i < sizeof(key); ++i) + key[i] = static_cast(i); + + const uint8_t plaintext[aes_block_length] = { + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, + 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}; + const uint8_t zero_iv[aes_block_length] = {}; + + // A single block under a zero vector is plain ECB, which is what the + // published vector covers. + uint8_t ciphertext[aes_block_length]; + const aes256_cbc cipher{key}; + cipher.encrypt(zero_iv, plaintext, ciphertext, sizeof(plaintext)); + BOOST_CHECK_EQUAL( + to_hex(ciphertext, sizeof(ciphertext)), + "8ea2b7ca516745bfeafc49904b496089"); + + uint8_t recovered[aes_block_length]; + cipher.decrypt(zero_iv, ciphertext, recovered, sizeof(ciphertext)); + BOOST_CHECK_EQUAL( + to_hex(recovered, sizeof(recovered)), + to_hex(plaintext, sizeof(plaintext))); +} + +BOOST_TEST_DECORATOR( + *utf::description("aes256_cbc round-trips multiple blocks")) +BOOST_AUTO_TEST_CASE(aes256_cbc__multiple_blocks__round_trip) +{ + uint8_t key[aes256_key_length]; + uint8_t iv[aes_block_length]; + for (size_t i = 0; i < sizeof(key); ++i) + key[i] = static_cast(i * 3); + for (size_t i = 0; i < sizeof(iv); ++i) + iv[i] = static_cast(0xa0 + i); + + std::vector plaintext(512); + for (size_t i = 0; i < plaintext.size(); ++i) + plaintext[i] = static_cast(i * 7); + + const aes256_cbc cipher{key}; + + std::vector ciphertext(plaintext.size()); + cipher.encrypt(iv, plaintext.data(), ciphertext.data(), plaintext.size()); + BOOST_CHECK(ciphertext != plaintext); + + // Decryption in place must work, as the codec relies on it. + std::vector buffer = ciphertext; + cipher.decrypt(iv, buffer.data(), buffer.data(), buffer.size()); + BOOST_CHECK(buffer == plaintext); +} + +BOOST_TEST_DECORATOR( + *utf::description("sqlcipher_codec round-trips a page losslessly")) +BOOST_AUTO_TEST_CASE(sqlcipher_codec__page_round_trip__is_lossless) +{ + const auto params = cheap_parameters(); + const sqlcipher_encryptor writer{"a passphrase", params}; + const sqlcipher_codec codec{"a passphrase", writer.salt(), params}; + + std::vector page(params.page_size, 0); + for (size_t i = 0; i < params.payload_size(); ++i) + page[i] = static_cast((i * 31) & 0xff); + + for (uint32_t page_number : {1u, 2u, 4096u}) + { + std::vector encrypted(params.page_size); + writer.encrypt_page(page_number, page.data(), encrypted.data()); + + std::vector decrypted(params.page_size); + codec.decrypt_page(page_number, encrypted.data(), decrypted.data()); + + // Page one carries the salt in place of the first sixteen bytes. + const auto offset = page_number == 1 ? sqlcipher_salt_length : 0; + BOOST_CHECK( + std::memcmp( + decrypted.data() + offset, page.data() + offset, + params.payload_size() - offset) == 0); + } +} + +BOOST_TEST_DECORATOR( + *utf::description("sqlcipher_codec rejects a tampered page")) +BOOST_AUTO_TEST_CASE(sqlcipher_codec__tampered_page__is_rejected) +{ + const auto params = cheap_parameters(); + const sqlcipher_encryptor writer{"a passphrase", params}; + const sqlcipher_codec codec{"a passphrase", writer.salt(), params}; + + std::vector page(params.page_size, 0x5a); + std::vector encrypted(params.page_size); + writer.encrypt_page(2, page.data(), encrypted.data()); + + encrypted[100] = static_cast(encrypted[100] ^ 0x01); + + std::vector decrypted(params.page_size); + BOOST_CHECK_THROW( + codec.decrypt_page(2, encrypted.data(), decrypted.data()), + sqlcipher_error); +} + +BOOST_TEST_DECORATOR( + *utf::description("sqlcipher_codec rejects a wrong passphrase")) +BOOST_AUTO_TEST_CASE(sqlcipher_codec__wrong_passphrase__is_rejected) +{ + const auto params = cheap_parameters(); + const sqlcipher_encryptor writer{"the right one", params}; + const sqlcipher_codec reader{"the wrong one", writer.salt(), params}; + + std::vector page(params.page_size, 0x11); + std::vector encrypted(params.page_size); + writer.encrypt_page(3, page.data(), encrypted.data()); + + std::vector decrypted(params.page_size); + BOOST_CHECK_THROW( + reader.decrypt_page(3, encrypted.data(), decrypted.data()), + sqlcipher_error); +} + +BOOST_TEST_DECORATOR( + *utf::description("sqlcipher_codec binds the page number into the tag")) +BOOST_AUTO_TEST_CASE(sqlcipher_codec__page_number__is_bound_into_the_tag) +{ + const auto params = cheap_parameters(); + const sqlcipher_encryptor writer{"a passphrase", params}; + const sqlcipher_codec codec{"a passphrase", writer.salt(), params}; + + std::vector page(params.page_size, 0x22); + std::vector encrypted(params.page_size); + writer.encrypt_page(7, page.data(), encrypted.data()); + + std::vector decrypted(params.page_size); + BOOST_CHECK_NO_THROW( + codec.decrypt_page(7, encrypted.data(), decrypted.data())); + + // The same bytes, read as a different page, must not verify. + BOOST_CHECK_THROW( + codec.decrypt_page(8, encrypted.data(), decrypted.data()), + sqlcipher_error); +} From 4acaa4ac118474562683a6b7562acf21ae73fd1a Mon Sep 17 00:00:00 2001 From: Illia Komsa Date: Mon, 31 Aug 2026 00:26:04 +0200 Subject: [PATCH 02/10] Speed up decryption of OneLibrary databases Reading an encrypted database was dominated by software AES and by copying, so several layers of that are removed: * AES uses the processor's own AES instructions where it has them (x86 AES-NI, the ARMv8 cryptographic extension), decided at run time and confined to `aes_hardware.cpp`, which the build compiles with the intrinsics enabled only where they compile at all. Decryption runs eight independent blocks at a time to keep the pipeline full. * The software fallback moves to fused round tables and to the equivalent inverse cipher, so both directions have the same shape. * SHA-512 keeps a sixteen-word wrapping schedule and unrolls its rounds by rotating variable names instead of shifting values. * HMAC keys absorb both padded blocks once, which halves key derivation over its 256,000 iterations and saves two compressions per page. * Page tags are computed over the ciphertext, IV and page number in place, rather than gathering each page into a scratch buffer. * The write-ahead log is read before the database, so a page the log replaces or truncates away is never decrypted at all. * Page decryption is spread over the available processors, pages being independent of one another. Tests gain the NIST SP 800-38A chaining vectors and run every cipher test over both implementations, so the tables are still exercised on a machine that would otherwise only ever use its AES instructions. --- .gitignore | 2 + CMakeLists.txt | 88 +++++ DjInteropConfig.cmake.in | 2 + src/djinterop/util/crypto/aes.cpp | 290 ++++++++++------ src/djinterop/util/crypto/aes.hpp | 44 ++- src/djinterop/util/crypto/aes_hardware.cpp | 321 +++++++++++++++++ src/djinterop/util/crypto/aes_hardware.hpp | 53 +++ src/djinterop/util/crypto/sha512.cpp | 181 ++++++---- src/djinterop/util/crypto/sha512.hpp | 35 ++ src/djinterop/util/crypto/sqlcipher_codec.cpp | 37 +- src/djinterop/util/crypto/sqlcipher_codec.hpp | 4 +- src/djinterop/util/crypto/sqlcipher_wal.cpp | 326 ++++++++++++------ src/djinterop/util/crypto/sqlcipher_wal.hpp | 5 + test/djinterop/util/crypto_test.cpp | 140 ++++++-- 14 files changed, 1221 insertions(+), 307 deletions(-) create mode 100644 src/djinterop/util/crypto/aes_hardware.cpp create mode 100644 src/djinterop/util/crypto/aes_hardware.hpp diff --git a/.gitignore b/.gitignore index 1e8863e..1088c6f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ compile_commands.json # CMake typical build dirs /cmake_build* /cmake-build* + +/build diff --git a/CMakeLists.txt b/CMakeLists.txt index 30a1f90..3167337 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,6 +50,82 @@ option(SYSTEM_SQLITE "Use system installation of SQLite" ON) option(EXPERIMENTAL_ENABLE_SQLCIPHER "Use SQLCipher in place of SQLite" OFF) option(SYSTEM_SQLITE_MODERN_CPP "Use system installation of sqlite_modern_cpp" OFF) + +# The processor's own AES instructions are worth two orders of magnitude over +# software AES when reading an encrypted OneLibrary database, so use them where +# the compiler can be persuaded to emit them. Whether the processor running the +# built library has them is a separate question, decided at run time; settled +# here is only whether the intrinsics compile, and with which flags. They are +# applied to `aes_hardware.cpp` alone, so no other translation unit can pick up +# an instruction the target may not have. +include(CheckCXXSourceCompiles) + +set(DJINTEROP_AES_X86_SOURCE " +#if defined(_MSC_VER) +#include +#else +#include +#endif +int main() +{ + __m128i x = _mm_setzero_si128(); + x = _mm_aesenc_si128(x, x); + x = _mm_aesenclast_si128(x, x); + x = _mm_aesdec_si128(x, x); + x = _mm_aesdeclast_si128(x, x); + return _mm_cvtsi128_si32(x); +} +") + +set(DJINTEROP_AES_ARM64_SOURCE " +#include +int main() +{ + uint8x16_t x = vdupq_n_u8(0); + x = vaesmcq_u8(vaeseq_u8(x, x)); + x = vaesimcq_u8(vaesdq_u8(x, x)); + return vgetq_lane_u8(x, 0); +} +") + +set(DJINTEROP_AES_INTRINSICS_DEFINE "") +set(DJINTEROP_AES_INTRINSICS_FLAGS "") + +# Try one architecture, unless another has already answered. Bare comes first: +# MSVC needs no flags at all, and neither does a compiler whose default target +# already includes the extension, such as Apple's on arm64. +macro(djinterop_try_aes arch source flags) + if(NOT DJINTEROP_AES_INTRINSICS_DEFINE) + check_cxx_source_compiles("${source}" DJINTEROP_AES_${arch}_BARE) + if(NOT DJINTEROP_AES_${arch}_BARE) + set(CMAKE_REQUIRED_FLAGS "${flags}") + check_cxx_source_compiles( + "${source}" DJINTEROP_AES_${arch}_FLAGGED) + unset(CMAKE_REQUIRED_FLAGS) + if(DJINTEROP_AES_${arch}_FLAGGED) + set(DJINTEROP_AES_INTRINSICS_FLAGS "${flags}") + endif() + endif() + if(DJINTEROP_AES_${arch}_BARE OR DJINTEROP_AES_${arch}_FLAGGED) + set(DJINTEROP_AES_INTRINSICS_DEFINE + "DJINTEROP_AES_INTRINSICS_${arch}") + endif() + endif() +endmacro() + +djinterop_try_aes(X86 "${DJINTEROP_AES_X86_SOURCE}" "-maes -msse2") +djinterop_try_aes(ARM64 "${DJINTEROP_AES_ARM64_SOURCE}" "-march=armv8-a+crypto") + +if(DJINTEROP_AES_INTRINSICS_DEFINE) + message(STATUS "Using AES instructions where available at run time") + set_source_files_properties( + src/djinterop/util/crypto/aes_hardware.cpp PROPERTIES + COMPILE_DEFINITIONS "${DJINTEROP_AES_INTRINSICS_DEFINE}" + COMPILE_FLAGS "${DJINTEROP_AES_INTRINSICS_FLAGS}") +else() + message(STATUS "No AES instructions for this target; using AES tables") +endif() + add_library( DjInterop include/djinterop/album_art.hpp @@ -224,6 +300,8 @@ add_library( src/djinterop/util/chrono.hpp src/djinterop/util/crypto/aes.cpp src/djinterop/util/crypto/aes.hpp + src/djinterop/util/crypto/aes_hardware.cpp + src/djinterop/util/crypto/aes_hardware.hpp src/djinterop/util/crypto/sha512.cpp src/djinterop/util/crypto/sha512.hpp src/djinterop/util/crypto/sqlcipher_codec.cpp @@ -263,6 +341,14 @@ target_include_directories( $ $) +# Decrypting the pages of a database is spread over the processors available, +# which is what the standard threading library is needed for. +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) +target_link_libraries( + DjInterop PRIVATE + Threads::Threads) + # Always rely on system installation of zlib. set(ZLIB_MIN_VERSION 1.2.8) find_package(ZLIB ${ZLIB_MIN_VERSION} REQUIRED) @@ -505,6 +591,7 @@ if (Boost_FOUND AND Boost_filesystem_FOUND AND Boost_system_FOUND) add_djinterop_test(onelibrary/ database_test) target_sources(onelibrary_database_test PRIVATE src/djinterop/util/crypto/aes.cpp + src/djinterop/util/crypto/aes_hardware.cpp src/djinterop/util/crypto/sha512.cpp src/djinterop/util/crypto/sqlcipher_codec.cpp src/djinterop/util/filesystem.cpp) @@ -516,6 +603,7 @@ if (Boost_FOUND AND Boost_filesystem_FOUND AND Boost_system_FOUND) add_djinterop_test(util/ crypto_test) target_sources(util_crypto_test PRIVATE src/djinterop/util/crypto/aes.cpp + src/djinterop/util/crypto/aes_hardware.cpp src/djinterop/util/crypto/sha512.cpp src/djinterop/util/crypto/sqlcipher_codec.cpp) else() diff --git a/DjInteropConfig.cmake.in b/DjInteropConfig.cmake.in index 22d88cf..a18c8ce 100644 --- a/DjInteropConfig.cmake.in +++ b/DjInteropConfig.cmake.in @@ -8,6 +8,8 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") include(CMakeFindDependencyMacro) find_dependency(ZLIB) +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_dependency(Threads) if(DJINTEROP_SYSTEM_DATE_H) find_dependency(date) endif() diff --git a/src/djinterop/util/crypto/aes.cpp b/src/djinterop/util/crypto/aes.cpp index d010edc..6406e32 100644 --- a/src/djinterop/util/crypto/aes.cpp +++ b/src/djinterop/util/crypto/aes.cpp @@ -17,13 +17,15 @@ #include "aes.hpp" +#include #include +#include "aes_hardware.hpp" + namespace djinterop::util::crypto { namespace { -constexpr int rounds = 14; constexpr int key_words = 8; inline uint8_t rotl8(uint8_t x, unsigned shift) noexcept @@ -38,30 +40,61 @@ inline uint8_t xtime(uint8_t x) noexcept } /// Multiply two elements of GF(2^8) modulo the AES polynomial. -inline uint8_t gmul(uint8_t a, uint8_t b) noexcept +constexpr uint8_t gmul(uint8_t a, uint8_t b) noexcept { uint8_t result = 0; while (b != 0) { if (b & 1) result ^= a; - a = xtime(a); + a = static_cast((a << 1) ^ ((a & 0x80) ? 0x1b : 0x00)); b = static_cast(b >> 1); } return result; } -/// The AES substitution box and its inverse. +inline uint32_t load_be32(const uint8_t* p) noexcept +{ + return (static_cast(p[0]) << 24) | + (static_cast(p[1]) << 16) | + (static_cast(p[2]) << 8) | static_cast(p[3]); +} + +inline void store_be32(uint8_t* p, uint32_t v) noexcept +{ + p[0] = static_cast(v >> 24); + p[1] = static_cast(v >> 16); + p[2] = static_cast(v >> 8); + p[3] = static_cast(v); +} + +/// Pack four field elements into a column, most significant byte first. +constexpr uint32_t column(uint8_t r0, uint8_t r1, uint8_t r2, uint8_t r3) +{ + return (static_cast(r0) << 24) | + (static_cast(r1) << 16) | + (static_cast(r2) << 8) | static_cast(r3); +} + +/// The substitution boxes, and the round tables built on top of them. /// -/// These are derived rather than tabulated: each entry is the affine transform -/// of the multiplicative inverse in GF(2^8), which the standard walk over -/// p = 3^i, q = 3^-i enumerates in a single pass. -struct substitution_tables +/// The boxes are derived rather than tabulated: each entry is the affine +/// transform of the multiplicative inverse in GF(2^8), which the standard walk +/// over p = 3^i, q = 3^-i enumerates in a single pass. +/// +/// The round tables fold substitution and column mixing together, so a round +/// costs four lookups and four exclusive-ors per column rather than the field +/// arithmetic the definition calls for. Only the first table of each set is +/// held: the other three are rotations of it, and a rotation is cheaper than a +/// second cache line. +struct aes_tables { uint8_t forward[256]; uint8_t inverse[256]; + uint32_t encrypt[256]; + uint32_t decrypt[256]; - substitution_tables() noexcept : forward{}, inverse{} + aes_tables() noexcept : forward{}, inverse{}, encrypt{}, decrypt{} { uint8_t p = 1; uint8_t q = 1; @@ -86,25 +119,129 @@ struct substitution_tables // Zero has no multiplicative inverse; it maps to the affine constant. forward[0] = 0x63; inverse[0x63] = 0x00; + + for (int i = 0; i < 256; ++i) + { + // The column MixColumns makes of a byte standing alone in row 0, + // and the one InvMixColumns makes of the same. + const auto s = forward[i]; + encrypt[i] = column(gmul(s, 2), s, s, gmul(s, 3)); + + const auto t = inverse[i]; + decrypt[i] = + column(gmul(t, 14), gmul(t, 9), gmul(t, 13), gmul(t, 11)); + } } }; -const substitution_tables& tables() noexcept +const aes_tables& tables() noexcept { - static const substitution_tables instance; + static const aes_tables instance; return instance; } +/// Extract the byte of a column that ShiftRows will place in a given row. +inline uint8_t row_of(uint32_t c, int row) noexcept +{ + return static_cast(c >> (24 - (8 * row))); +} + +/// Apply InvMixColumns to a round key in place, turning the forward schedule +/// into the one the equivalent inverse cipher wants. +void inverse_mix_columns(uint8_t* key) noexcept +{ + for (int i = 0; i < 4; ++i) + { + auto* c = key + (4 * i); + const uint8_t a0 = c[0], a1 = c[1], a2 = c[2], a3 = c[3]; + c[0] = static_cast( + gmul(a0, 14) ^ gmul(a1, 11) ^ gmul(a2, 13) ^ gmul(a3, 9)); + c[1] = static_cast( + gmul(a0, 9) ^ gmul(a1, 14) ^ gmul(a2, 11) ^ gmul(a3, 13)); + c[2] = static_cast( + gmul(a0, 13) ^ gmul(a1, 9) ^ gmul(a2, 14) ^ gmul(a3, 11)); + c[3] = static_cast( + gmul(a0, 11) ^ gmul(a1, 13) ^ gmul(a2, 9) ^ gmul(a3, 14)); + } +} + +/// The number of 32-bit words in a key schedule. +constexpr int schedule_words = 4 * (aes256_rounds + 1); + +/// Which way ShiftRows moves the rows along. +constexpr int forwards = 1; +constexpr int backwards = -1; + +/// Unpack a schedule into words once, rather than reassembling four bytes at +/// every use: a page is thousands of blocks, and a schedule is sixty words. +void unpack_schedule(const uint8_t* keys, uint32_t* words) noexcept +{ + for (int i = 0; i < schedule_words; ++i) + words[i] = load_be32(keys + (4 * i)); +} + +/// One block through the round tables, in either direction. +/// +/// Once decryption goes through the equivalent inverse cipher the two +/// directions have the same shape, and differ only in which tables they read +/// and which way the rows shift. +template +void transform_block( + const uint32_t* keys, const uint32_t* table, const uint8_t* box, + const uint8_t* input, uint8_t* output) noexcept +{ + // ShiftRows draws row r of an output column from the column r steps away, + // forwards when encrypting and backwards when decrypting. + const auto from = [](int c, int row) + { return (c + (direction * row) + 4) & 3; }; + + uint32_t s[4]; + for (int i = 0; i < 4; ++i) + s[i] = load_be32(input + (4 * i)) ^ keys[i]; + + for (int round = 1; round < aes256_rounds; ++round) + { + const auto* rk = keys + (4 * round); + + uint32_t u[4]; + for (int c = 0; c < 4; ++c) + { + u[c] = table[row_of(s[from(c, 0)], 0)] ^ + std::rotr(table[row_of(s[from(c, 1)], 1)], 8) ^ + std::rotr(table[row_of(s[from(c, 2)], 2)], 16) ^ + std::rotr(table[row_of(s[from(c, 3)], 3)], 24) ^ rk[c]; + } + + std::memcpy(s, u, sizeof(s)); + } + + // The last round leaves out the column mixing, so it reads the box direct. + const auto* rk = keys + (4 * aes256_rounds); + for (int c = 0; c < 4; ++c) + { + const auto substituted = column( + box[row_of(s[from(c, 0)], 0)], box[row_of(s[from(c, 1)], 1)], + box[row_of(s[from(c, 2)], 2)], box[row_of(s[from(c, 3)], 3)]); + + store_be32(output + (4 * c), substituted ^ rk[c]); + } +} + } // anonymous namespace -aes256_cbc::aes256_cbc(const uint8_t* key) noexcept : round_keys_{} +aes256_cbc::aes256_cbc( + const uint8_t* key, aes_implementation implementation) noexcept : + round_keys_{}, inverse_round_keys_{}, + hardware_{ + implementation == aes_implementation::automatic && + hardware::aes_available()} { const auto& sbox = tables().forward; std::memcpy(round_keys_.data(), key, aes256_key_length); uint8_t rcon = 1; - for (int word = key_words; word < 4 * (rounds + 1); ++word) + for (int word = key_words; word < 4 * (aes256_rounds + 1); ++word) { uint8_t temp[4]; std::memcpy(temp, round_keys_.data() + (4 * (word - 1)), 4); @@ -131,102 +268,39 @@ aes256_cbc::aes256_cbc(const uint8_t* key) noexcept : round_keys_{} round_keys_[(4 * (word - key_words)) + i] ^ temp[i]; } } -} - -void aes256_cbc::encrypt_block( - const uint8_t* input, uint8_t* output) const noexcept -{ - const auto& sbox = tables().forward; - - uint8_t state[16]; - for (int i = 0; i < 16; ++i) - state[i] = input[i] ^ round_keys_[i]; - for (int round = 1; round <= rounds; ++round) + // Reverse the schedule, and fold InvMixColumns into every round key but + // the two on the ends, which are only ever added and never mixed. + for (int round = 0; round <= aes256_rounds; ++round) { - for (auto& byte : state) - byte = sbox[byte]; - - // ShiftRows: the state is column-major, so row r is bytes r, r+4, ... - uint8_t shifted[16]; - for (int column = 0; column < 4; ++column) - for (int row = 0; row < 4; ++row) - shifted[(4 * column) + row] = - state[(4 * ((column + row) % 4)) + row]; - std::memcpy(state, shifted, 16); - - if (round != rounds) - { - for (int column = 0; column < 4; ++column) - { - auto* c = state + (4 * column); - const auto sum = - static_cast(c[0] ^ c[1] ^ c[2] ^ c[3]); - const auto c0 = c[0]; - c[0] ^= static_cast(sum ^ xtime(c[0] ^ c[1])); - c[1] ^= static_cast(sum ^ xtime(c[1] ^ c[2])); - c[2] ^= static_cast(sum ^ xtime(c[2] ^ c[3])); - c[3] ^= static_cast(sum ^ xtime(c[3] ^ c0)); - } - } - - for (int i = 0; i < 16; ++i) - state[i] ^= round_keys_[(16 * round) + i]; + auto* destination = + inverse_round_keys_.data() + (aes_block_length * round); + std::memcpy( + destination, + round_keys_.data() + (aes_block_length * (aes256_rounds - round)), + aes_block_length); + + if (round != 0 && round != aes256_rounds) + inverse_mix_columns(destination); } - - std::memcpy(output, state, 16); } -void aes256_cbc::decrypt_block( - const uint8_t* input, uint8_t* output) const noexcept +void aes256_cbc::encrypt( + const uint8_t* iv, const uint8_t* input, uint8_t* output, + size_t length) const noexcept { - const auto& rsbox = tables().inverse; - - uint8_t state[16]; - for (int i = 0; i < 16; ++i) - state[i] = input[i] ^ round_keys_[(16 * rounds) + i]; - - for (int round = rounds - 1; round >= 0; --round) + if (hardware_) { - // InvShiftRows. - uint8_t shifted[16]; - for (int column = 0; column < 4; ++column) - for (int row = 0; row < 4; ++row) - shifted[(4 * ((column + row) % 4)) + row] = - state[(4 * column) + row]; - std::memcpy(state, shifted, 16); - - for (auto& byte : state) - byte = rsbox[byte]; - - for (int i = 0; i < 16; ++i) - state[i] ^= round_keys_[(16 * round) + i]; - - if (round != 0) - { - for (int column = 0; column < 4; ++column) - { - auto* c = state + (4 * column); - const uint8_t a0 = c[0], a1 = c[1], a2 = c[2], a3 = c[3]; - c[0] = static_cast( - gmul(a0, 14) ^ gmul(a1, 11) ^ gmul(a2, 13) ^ gmul(a3, 9)); - c[1] = static_cast( - gmul(a0, 9) ^ gmul(a1, 14) ^ gmul(a2, 11) ^ gmul(a3, 13)); - c[2] = static_cast( - gmul(a0, 13) ^ gmul(a1, 9) ^ gmul(a2, 14) ^ gmul(a3, 11)); - c[3] = static_cast( - gmul(a0, 11) ^ gmul(a1, 13) ^ gmul(a2, 9) ^ gmul(a3, 14)); - } - } + hardware::aes256_cbc_encrypt( + round_keys_.data(), iv, input, output, length); + return; } - std::memcpy(output, state, 16); -} + const auto& t = tables(); + + uint32_t schedule[schedule_words]; + unpack_schedule(round_keys_.data(), schedule); -void aes256_cbc::encrypt( - const uint8_t* iv, const uint8_t* input, uint8_t* output, - size_t length) const noexcept -{ uint8_t chain[aes_block_length]; std::memcpy(chain, iv, aes_block_length); @@ -236,7 +310,8 @@ void aes256_cbc::encrypt( for (size_t i = 0; i < aes_block_length; ++i) block[i] = input[offset + i] ^ chain[i]; - encrypt_block(block, output + offset); + transform_block( + schedule, t.encrypt, t.forward, block, output + offset); std::memcpy(chain, output + offset, aes_block_length); } } @@ -245,6 +320,18 @@ void aes256_cbc::decrypt( const uint8_t* iv, const uint8_t* input, uint8_t* output, size_t length) const noexcept { + if (hardware_) + { + hardware::aes256_cbc_decrypt( + inverse_round_keys_.data(), iv, input, output, length); + return; + } + + const auto& t = tables(); + + uint32_t schedule[schedule_words]; + unpack_schedule(inverse_round_keys_.data(), schedule); + uint8_t chain[aes_block_length]; std::memcpy(chain, iv, aes_block_length); @@ -254,7 +341,8 @@ void aes256_cbc::decrypt( uint8_t next_chain[aes_block_length]; std::memcpy(next_chain, input + offset, aes_block_length); - decrypt_block(input + offset, output + offset); + transform_block( + schedule, t.decrypt, t.inverse, input + offset, output + offset); for (size_t i = 0; i < aes_block_length; ++i) output[offset + i] ^= chain[i]; diff --git a/src/djinterop/util/crypto/aes.hpp b/src/djinterop/util/crypto/aes.hpp index bfa30eb..954825b 100644 --- a/src/djinterop/util/crypto/aes.hpp +++ b/src/djinterop/util/crypto/aes.hpp @@ -26,18 +26,44 @@ namespace djinterop::util::crypto constexpr size_t aes_block_length = 16; constexpr size_t aes256_key_length = 32; +/// Number of rounds in AES-256, and hence one less than the number of round +/// keys that its schedule expands to. +constexpr int aes256_rounds = 14; + +constexpr size_t aes256_schedule_length = + aes_block_length * (aes256_rounds + 1); + +/// Which implementation an instance should use. +enum class aes_implementation +{ + /// The processor's AES instructions where it has them, tables otherwise. + automatic, + + /// The tables, whatever the processor offers. Nothing in the library asks + /// for this; it lets tests reach the fallback on a machine that would + /// otherwise never run it. + tabulated, +}; + /// AES-256 in cipher block chaining mode, as specified by FIPS 197 and /// NIST SP 800-38A. /// /// No padding scheme is applied: input lengths must be a whole number of /// blocks. SQLCipher pages are always block-aligned by construction, which is /// why a padding mode is not needed. +/// +/// Where the processor has AES instructions they are used, and tables +/// otherwise. Neither resists timing analysis, and the tables plainly do not; +/// the passphrase of a OneLibrary database is a constant compiled into +/// rekordbox rather than a secret, so there is nothing to learn from it. class aes256_cbc { public: /// Construct a cipher for a given key, which must be /// `aes256_key_length` bytes long. - explicit aes256_cbc(const uint8_t* key) noexcept; + explicit aes256_cbc( + const uint8_t* key, aes_implementation implementation = + aes_implementation::automatic) noexcept; /// Encrypt `length` bytes from `input` into `output`, which may alias /// `input`. `length` must be a multiple of the AES block length. @@ -54,11 +80,19 @@ class aes256_cbc size_t length) const noexcept; private: - void encrypt_block(const uint8_t* input, uint8_t* output) const noexcept; - void decrypt_block(const uint8_t* input, uint8_t* output) const noexcept; - /// Expanded key schedule: 15 round keys of 16 bytes each. - std::array round_keys_; + alignas(16) std::array round_keys_; + + /// The schedule of the equivalent inverse cipher: the round keys reversed, + /// all but the outermost two passed through InvMixColumns. + /// + /// Folding that transform into the keys lets the inverse rounds take the + /// same shape as the forward ones, which is what both the tables and the + /// processor's AES instructions expect. + alignas(16) std::array inverse_round_keys_; + + /// Whether the processor this program is running on has AES instructions. + bool hardware_; }; } // namespace djinterop::util::crypto diff --git a/src/djinterop/util/crypto/aes_hardware.cpp b/src/djinterop/util/crypto/aes_hardware.cpp new file mode 100644 index 0000000..c254f5d --- /dev/null +++ b/src/djinterop/util/crypto/aes_hardware.cpp @@ -0,0 +1,321 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#include "aes_hardware.hpp" + +#include "aes.hpp" + +// One of these is defined by the build system, if it found a way to compile the +// intrinsics for the target. Anything else falls back to the tables in +// `aes.cpp`. +#if defined(DJINTEROP_AES_INTRINSICS_X86) +#if defined(_MSC_VER) +#include +#else +#include +#endif +#elif defined(DJINTEROP_AES_INTRINSICS_ARM64) +#include +#if defined(_WIN32) +#include +#elif defined(__linux__) +#include +#if !defined(HWCAP_AES) +#define HWCAP_AES (1 << 3) +#endif +#elif defined(__FreeBSD__) +#include +#include +#endif +#endif + +#if defined(DJINTEROP_AES_INTRINSICS_X86) || \ + defined(DJINTEROP_AES_INTRINSICS_ARM64) +#define DJINTEROP_AES_INTRINSICS 1 +#endif + +namespace djinterop::util::crypto::hardware +{ +#if defined(DJINTEROP_AES_INTRINSICS) + +namespace +{ +// Each architecture supplies the same handful of operations, over which the +// chaining below is written once. Rounds are counted as x86 counts them: +// AArch64 adds its round key at the start of a round rather than the end, so +// its idiom looks a round out of step while doing the same work. + +#if defined(DJINTEROP_AES_INTRINSICS_X86) + +using block = __m128i; + +inline block load(const uint8_t* p) noexcept +{ + return _mm_loadu_si128(reinterpret_cast(p)); +} + +inline void store(uint8_t* p, block x) noexcept +{ + _mm_storeu_si128(reinterpret_cast<__m128i*>(p), x); +} + +inline block block_xor(block a, block b) noexcept +{ + return _mm_xor_si128(a, b); +} + +inline block encrypt_first(block x, const block* rk) noexcept +{ + return _mm_xor_si128(x, rk[0]); +} + +inline block encrypt_round(block x, const block* rk, int round) noexcept +{ + return _mm_aesenc_si128(x, rk[round + 1]); +} + +inline block encrypt_last(block x, const block* rk) noexcept +{ + return _mm_aesenclast_si128(x, rk[aes256_rounds]); +} + +inline block decrypt_first(block x, const block* dk) noexcept +{ + return _mm_xor_si128(x, dk[0]); +} + +inline block decrypt_round(block x, const block* dk, int round) noexcept +{ + return _mm_aesdec_si128(x, dk[round + 1]); +} + +inline block decrypt_last(block x, const block* dk) noexcept +{ + return _mm_aesdeclast_si128(x, dk[aes256_rounds]); +} + +#else + +using block = uint8x16_t; + +inline block load(const uint8_t* p) noexcept +{ + return vld1q_u8(p); +} + +inline void store(uint8_t* p, block x) noexcept +{ + vst1q_u8(p, x); +} + +inline block block_xor(block a, block b) noexcept +{ + return veorq_u8(a, b); +} + +inline block encrypt_first(block x, const block*) noexcept +{ + return x; +} + +inline block encrypt_round(block x, const block* rk, int round) noexcept +{ + return vaesmcq_u8(vaeseq_u8(x, rk[round])); +} + +inline block encrypt_last(block x, const block* rk) noexcept +{ + return veorq_u8(vaeseq_u8(x, rk[aes256_rounds - 1]), rk[aes256_rounds]); +} + +inline block decrypt_first(block x, const block*) noexcept +{ + return x; +} + +inline block decrypt_round(block x, const block* dk, int round) noexcept +{ + return vaesimcq_u8(vaesdq_u8(x, dk[round])); +} + +inline block decrypt_last(block x, const block* dk) noexcept +{ + return veorq_u8(vaesdq_u8(x, dk[aes256_rounds - 1]), dk[aes256_rounds]); +} + +#endif + +/// Rounds between the first and the last, of which both directions have the +/// same number. +constexpr int middle_rounds = aes256_rounds - 1; + +/// Blocks decrypted together. +/// +/// The AES instructions take several cycles to yield a result but accept a new +/// block every cycle, so independent blocks run several times faster than a +/// dependent chain does. Eight covers the latency without exhausting the +/// sixteen vector registers both architectures have. +constexpr size_t lanes = 8; + +void load_schedule(const uint8_t* keys, block* out) noexcept +{ + for (int i = 0; i <= aes256_rounds; ++i) + out[i] = load(keys + (aes_block_length * i)); +} + +inline block decrypt_block(block x, const block* dk) noexcept +{ + x = decrypt_first(x, dk); + for (int round = 0; round < middle_rounds; ++round) + x = decrypt_round(x, dk, round); + + return decrypt_last(x, dk); +} + +} // anonymous namespace + +void aes256_cbc_encrypt( + const uint8_t* round_keys, const uint8_t* iv, const uint8_t* input, + uint8_t* output, size_t length) noexcept +{ + block rk[aes256_rounds + 1]; + load_schedule(round_keys, rk); + + auto chain = load(iv); + for (size_t offset = 0; offset < length; offset += aes_block_length) + { + auto x = encrypt_first(block_xor(load(input + offset), chain), rk); + for (int round = 0; round < middle_rounds; ++round) + x = encrypt_round(x, rk, round); + + chain = encrypt_last(x, rk); + store(output + offset, chain); + } +} + +void aes256_cbc_decrypt( + const uint8_t* inverse_round_keys, const uint8_t* iv, const uint8_t* input, + uint8_t* output, size_t length) noexcept +{ + block dk[aes256_rounds + 1]; + load_schedule(inverse_round_keys, dk); + + auto chain = load(iv); + size_t offset = 0; + + constexpr size_t stride = lanes * aes_block_length; + for (; offset + stride <= length; offset += stride) + { + // Every block of the group is read before any is written, so the + // ciphertext each one chains with survives an output that aliases the + // input. + block ciphertext[lanes]; + block x[lanes]; + for (size_t lane = 0; lane < lanes; ++lane) + { + ciphertext[lane] = load(input + offset + (lane * aes_block_length)); + x[lane] = decrypt_first(ciphertext[lane], dk); + } + + for (int round = 0; round < middle_rounds; ++round) + for (auto& lane : x) + lane = decrypt_round(lane, dk, round); + + for (auto& lane : x) + lane = decrypt_last(lane, dk); + + x[0] = block_xor(x[0], chain); + for (size_t lane = 1; lane < lanes; ++lane) + x[lane] = block_xor(x[lane], ciphertext[lane - 1]); + + chain = ciphertext[lanes - 1]; + for (size_t lane = 0; lane < lanes; ++lane) + store(output + offset + (lane * aes_block_length), x[lane]); + } + + for (; offset < length; offset += aes_block_length) + { + const auto ciphertext = load(input + offset); + store(output + offset, block_xor(decrypt_block(ciphertext, dk), chain)); + chain = ciphertext; + } +} + +bool aes_available() noexcept +{ +#if defined(DJINTEROP_AES_INTRINSICS_X86) && defined(_MSC_VER) + // AES-NI is bit 25 of ECX for CPUID leaf 1. + static const bool available = [] + { + int registers[4] = {0, 0, 0, 0}; + __cpuid(registers, 1); + return (registers[2] & (1 << 25)) != 0; + }(); + return available; +#elif defined(DJINTEROP_AES_INTRINSICS_X86) + static const bool available = __builtin_cpu_supports("aes"); + return available; +#elif defined(__APPLE__) + // Every processor Apple has shipped in an arm64 device implements the + // cryptographic extension, and the platform guarantees it. + return true; +#elif defined(_WIN32) + static const bool available = + IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != + FALSE; + return available; +#elif defined(__linux__) + static const bool available = (getauxval(AT_HWCAP) & HWCAP_AES) != 0; + return available; +#elif defined(__FreeBSD__) + static const bool available = [] + { + unsigned long capabilities = 0; + if (elf_aux_info(AT_HWCAP, &capabilities, sizeof(capabilities)) != 0) + return false; + + return (capabilities & HWCAP_AES) != 0; + }(); + return available; +#else + // With no way to ask, assume not: the fallback is correct, only slower. + return false; +#endif +} + +#else + +bool aes_available() noexcept +{ + return false; +} + +// Never reached: `aes_available` says so, and every caller asks first. + +void aes256_cbc_encrypt( + const uint8_t*, const uint8_t*, const uint8_t*, uint8_t*, size_t) noexcept +{ +} + +void aes256_cbc_decrypt( + const uint8_t*, const uint8_t*, const uint8_t*, uint8_t*, size_t) noexcept +{ +} + +#endif + +} // namespace djinterop::util::crypto::hardware diff --git a/src/djinterop/util/crypto/aes_hardware.hpp b/src/djinterop/util/crypto/aes_hardware.hpp new file mode 100644 index 0000000..3a4fe28 --- /dev/null +++ b/src/djinterop/util/crypto/aes_hardware.hpp @@ -0,0 +1,53 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include +#include + +/// The processor's own AES instructions, where it has them. +/// +/// x86 has had AES-NI since 2010 and AArch64 the ARMv8 cryptographic extension +/// from the start, so in practice these are what runs, and the tables in +/// `aes.cpp` are the fallback. Whether the instructions are there is a +/// property of the processor rather than of the build, so it is asked at run +/// time and this is the only translation unit compiled with them enabled. +namespace djinterop::util::crypto::hardware +{ +/// Whether the processor running this program has AES instructions. The +/// underlying query is made once and remembered. +[[nodiscard]] bool aes_available() noexcept; + +/// Encrypt in cipher block chaining mode. `length` is a whole number of +/// blocks, `output` may alias `input`, and `round_keys` is the forward +/// schedule of 15 round keys. +void aes256_cbc_encrypt( + const uint8_t* round_keys, const uint8_t* iv, const uint8_t* input, + uint8_t* output, size_t length) noexcept; + +/// Decrypt in cipher block chaining mode, taking the schedule of the +/// equivalent inverse cipher, which is the form both instruction sets expect. +/// +/// Chaining constrains encryption to one block at a time, but not decryption: +/// a block needs only its own ciphertext and the one before it, so several go +/// at once here to keep the pipeline of the AES instructions full. +void aes256_cbc_decrypt( + const uint8_t* inverse_round_keys, const uint8_t* iv, const uint8_t* input, + uint8_t* output, size_t length) noexcept; + +} // namespace djinterop::util::crypto::hardware diff --git a/src/djinterop/util/crypto/sha512.cpp b/src/djinterop/util/crypto/sha512.cpp index 2947727..85ab43f 100644 --- a/src/djinterop/util/crypto/sha512.cpp +++ b/src/djinterop/util/crypto/sha512.cpp @@ -59,6 +59,63 @@ inline uint64_t rotr(uint64_t x, unsigned n) noexcept return (x >> n) | (x << (64 - n)); } +/// Extend the message schedule in place. Only the last sixteen words are ever +/// needed, so they live in a window that wraps rather than in the array of +/// eighty the specification describes. +inline uint64_t extend(uint64_t* w, int j) noexcept +{ + const auto x = w[(j + 1) & 15]; + const auto y = w[(j + 14) & 15]; + w[j] += (rotr(x, 1) ^ rotr(x, 8) ^ (x >> 7)) + w[(j + 9) & 15] + + (rotr(y, 19) ^ rotr(y, 61) ^ (y >> 6)); + return w[j]; +} + +/// One round, over working variables named in the order the round expects them. +/// +/// The specification shifts the eight variables along by one each round; the +/// caller below rotates their *names* instead, which is free. Sixteen rounds +/// are two whole turns of the eight, so a block ends with every name back +/// where it started. +#define DJINTEROP_SHA512_ROUND(a, b, c, d, e, f, g, h, word, constant) \ + do \ + { \ + const uint64_t t1 = \ + (h) + (rotr((e), 14) ^ rotr((e), 18) ^ rotr((e), 41)) + \ + (((e) & (f)) ^ (~(e) & (g))) + (constant) + (word); \ + const uint64_t t2 = (rotr((a), 28) ^ rotr((a), 34) ^ rotr((a), 39)) + \ + (((a) & (b)) ^ ((a) & (c)) ^ ((b) & (c))); \ + (d) += t1; \ + (h) = t1 + t2; \ + } while (false) + +#define DJINTEROP_SHA512_BLOCK(word) \ + do \ + { \ + DJINTEROP_SHA512_ROUND(a, b, c, d, e, f, g, h, word(0), k[i + 0]); \ + DJINTEROP_SHA512_ROUND(h, a, b, c, d, e, f, g, word(1), k[i + 1]); \ + DJINTEROP_SHA512_ROUND(g, h, a, b, c, d, e, f, word(2), k[i + 2]); \ + DJINTEROP_SHA512_ROUND(f, g, h, a, b, c, d, e, word(3), k[i + 3]); \ + DJINTEROP_SHA512_ROUND(e, f, g, h, a, b, c, d, word(4), k[i + 4]); \ + DJINTEROP_SHA512_ROUND(d, e, f, g, h, a, b, c, word(5), k[i + 5]); \ + DJINTEROP_SHA512_ROUND(c, d, e, f, g, h, a, b, word(6), k[i + 6]); \ + DJINTEROP_SHA512_ROUND(b, c, d, e, f, g, h, a, word(7), k[i + 7]); \ + DJINTEROP_SHA512_ROUND(a, b, c, d, e, f, g, h, word(8), k[i + 8]); \ + DJINTEROP_SHA512_ROUND(h, a, b, c, d, e, f, g, word(9), k[i + 9]); \ + DJINTEROP_SHA512_ROUND(g, h, a, b, c, d, e, f, word(10), k[i + 10]); \ + DJINTEROP_SHA512_ROUND(f, g, h, a, b, c, d, e, word(11), k[i + 11]); \ + DJINTEROP_SHA512_ROUND(e, f, g, h, a, b, c, d, word(12), k[i + 12]); \ + DJINTEROP_SHA512_ROUND(d, e, f, g, h, a, b, c, word(13), k[i + 13]); \ + DJINTEROP_SHA512_ROUND(c, d, e, f, g, h, a, b, word(14), k[i + 14]); \ + DJINTEROP_SHA512_ROUND(b, c, d, e, f, g, h, a, word(15), k[i + 15]); \ + i += 16; \ + } while (false) + +/// The first sixteen rounds read the block as it arrives; the rest extend the +/// schedule a word at a time, just before the round that consumes it. +#define DJINTEROP_SHA512_LOADED(j) w[j] +#define DJINTEROP_SHA512_EXTENDED(j) extend(w, j) + inline uint64_t load_be64(const uint8_t* p) noexcept { uint64_t v = 0; @@ -88,39 +145,19 @@ sha512::sha512() noexcept : void sha512::compress(const uint8_t* block) noexcept { - uint64_t w[80]; - for (int i = 0; i < 16; ++i) - w[i] = load_be64(block + (i * 8)); - for (int i = 16; i < 80; ++i) - { - const auto s0 = - rotr(w[i - 15], 1) ^ rotr(w[i - 15], 8) ^ (w[i - 15] >> 7); - const auto s1 = - rotr(w[i - 2], 19) ^ rotr(w[i - 2], 61) ^ (w[i - 2] >> 6); - w[i] = w[i - 16] + s0 + w[i - 7] + s1; - } + uint64_t w[16]; + for (int j = 0; j < 16; ++j) + w[j] = load_be64(block + (j * 8)); auto a = state_[0], b = state_[1], c = state_[2], d = state_[3]; auto e = state_[4], f = state_[5], g = state_[6], h = state_[7]; - for (int i = 0; i < 80; ++i) - { - const auto s1 = rotr(e, 14) ^ rotr(e, 18) ^ rotr(e, 41); - const auto ch = (e & f) ^ (~e & g); - const auto temp1 = h + s1 + ch + k[i] + w[i]; - const auto s0 = rotr(a, 28) ^ rotr(a, 34) ^ rotr(a, 39); - const auto maj = (a & b) ^ (a & c) ^ (b & c); - const auto temp2 = s0 + maj; - - h = g; - g = f; - f = e; - e = d + temp1; - d = c; - c = b; - b = a; - a = temp1 + temp2; - } + int i = 0; + DJINTEROP_SHA512_BLOCK(DJINTEROP_SHA512_LOADED); + DJINTEROP_SHA512_BLOCK(DJINTEROP_SHA512_EXTENDED); + DJINTEROP_SHA512_BLOCK(DJINTEROP_SHA512_EXTENDED); + DJINTEROP_SHA512_BLOCK(DJINTEROP_SHA512_EXTENDED); + DJINTEROP_SHA512_BLOCK(DJINTEROP_SHA512_EXTENDED); state_[0] += a; state_[1] += b; @@ -193,9 +230,7 @@ sha512_digest sha512::hash(const uint8_t* data, size_t length) noexcept return h.finalise(); } -sha512_digest hmac_sha512( - const uint8_t* key, size_t key_length, const uint8_t* data, - size_t data_length) noexcept +hmac_sha512_key::hmac_sha512_key(const uint8_t* key, size_t key_length) noexcept { std::array padded{}; if (key_length > sha512_block_length) @@ -208,23 +243,42 @@ sha512_digest hmac_sha512( std::copy(key, key + key_length, padded.begin()); } - std::array inner_pad{}; - std::array outer_pad{}; + std::array pad{}; for (size_t i = 0; i < sha512_block_length; ++i) - { - inner_pad[i] = static_cast(padded[i] ^ 0x36); - outer_pad[i] = static_cast(padded[i] ^ 0x5c); - } + pad[i] = static_cast(padded[i] ^ 0x36); + inner_.update(pad.data(), pad.size()); - sha512 inner; - inner.update(inner_pad.data(), inner_pad.size()); - inner.update(data, data_length); - const auto inner_digest = inner.finalise(); + for (size_t i = 0; i < sha512_block_length; ++i) + pad[i] = static_cast(padded[i] ^ 0x5c); + outer_.update(pad.data(), pad.size()); +} - sha512 outer; - outer.update(outer_pad.data(), outer_pad.size()); - outer.update(inner_digest.data(), inner_digest.size()); - return outer.finalise(); +hmac_sha512_stream::hmac_sha512_stream(const hmac_sha512_key& key) noexcept : + inner_{key.inner_}, outer_{key.outer_} +{ +} + +void hmac_sha512_stream::update(const uint8_t* data, size_t length) noexcept +{ + inner_.update(data, length); +} + +sha512_digest hmac_sha512_stream::finalise() noexcept +{ + const auto inner_digest = inner_.finalise(); + outer_.update(inner_digest.data(), inner_digest.size()); + return outer_.finalise(); +} + +sha512_digest hmac_sha512( + const uint8_t* key, size_t key_length, const uint8_t* data, + size_t data_length) noexcept +{ + const hmac_sha512_key prepared{key, key_length}; + + hmac_sha512_stream stream{prepared}; + stream.update(data, data_length); + return stream.finalise(); } std::vector pbkdf2_hmac_sha512( @@ -234,28 +288,34 @@ std::vector pbkdf2_hmac_sha512( if (iterations == 0) throw std::invalid_argument{"PBKDF2 requires at least one iteration"}; + // Every iteration below authenticates under the same key, so its padded + // blocks are absorbed once here rather than a quarter of a million times. + const hmac_sha512_key prf{password, password_length}; + std::vector output; output.reserve(length); - std::vector block; - block.reserve(salt_length + 4); - for (uint32_t index = 1; output.size() < length; ++index) { // U_1 = PRF(password, salt || INT_BE32(index)) - block.assign(salt, salt + salt_length); - block.push_back(static_cast(index >> 24)); - block.push_back(static_cast(index >> 16)); - block.push_back(static_cast(index >> 8)); - block.push_back(static_cast(index)); - - auto u = - hmac_sha512(password, password_length, block.data(), block.size()); + const uint8_t counter[4] = { + static_cast(index >> 24), + static_cast(index >> 16), static_cast(index >> 8), + static_cast(index)}; + + hmac_sha512_stream first{prf}; + first.update(salt, salt_length); + first.update(counter, sizeof(counter)); + + auto u = first.finalise(); auto accumulator = u; for (uint32_t i = 1; i < iterations; ++i) { - u = hmac_sha512(password, password_length, u.data(), u.size()); + hmac_sha512_stream next{prf}; + next.update(u.data(), u.size()); + u = next.finalise(); + for (size_t j = 0; j < accumulator.size(); ++j) accumulator[j] ^= u[j]; } @@ -269,3 +329,8 @@ std::vector pbkdf2_hmac_sha512( } } // namespace djinterop::util::crypto + +#undef DJINTEROP_SHA512_ROUND +#undef DJINTEROP_SHA512_BLOCK +#undef DJINTEROP_SHA512_LOADED +#undef DJINTEROP_SHA512_EXTENDED diff --git a/src/djinterop/util/crypto/sha512.hpp b/src/djinterop/util/crypto/sha512.hpp index b26635d..4f4380a 100644 --- a/src/djinterop/util/crypto/sha512.hpp +++ b/src/djinterop/util/crypto/sha512.hpp @@ -51,6 +51,41 @@ class sha512 uint64_t total_length_; }; +/// A key prepared for repeated HMAC-SHA-512. +/// +/// Both padded key blocks are absorbed once, here, so a message afterwards +/// costs only the compressions its own bytes call for. That halves key +/// derivation, and every page of a database is checked under the one key. +class hmac_sha512_key +{ +public: + hmac_sha512_key(const uint8_t* key, size_t key_length) noexcept; + +private: + friend class hmac_sha512_stream; + + sha512 inner_; + sha512 outer_; +}; + +/// One message authenticated under a prepared key. A stream holds no +/// reference to the key, so streams built from one key on several threads do +/// not interfere. +class hmac_sha512_stream +{ +public: + explicit hmac_sha512_stream(const hmac_sha512_key& key) noexcept; + + void update(const uint8_t* data, size_t length) noexcept; + + /// Finalise the tag. The object must not be reused afterwards. + [[nodiscard]] sha512_digest finalise() noexcept; + +private: + sha512 inner_; + sha512 outer_; +}; + /// Compute HMAC-SHA-512, as specified by RFC 2104. sha512_digest hmac_sha512( const uint8_t* key, size_t key_length, const uint8_t* data, diff --git a/src/djinterop/util/crypto/sqlcipher_codec.cpp b/src/djinterop/util/crypto/sqlcipher_codec.cpp index e98b549..fafa3d2 100644 --- a/src/djinterop/util/crypto/sqlcipher_codec.cpp +++ b/src/djinterop/util/crypto/sqlcipher_codec.cpp @@ -39,11 +39,11 @@ std::vector derive_page_key( salt.data(), salt.size(), iterations, page_key_length); } -/// Derive the HMAC key. +/// Derive the HMAC key, and prepare it for the pages checked under it. /// /// It comes from the *page key*, not the passphrase, using the database salt /// with every byte XORed by 0x3a. -std::vector derive_hmac_key( +hmac_sha512_key make_hmac_key( const std::vector& page_key, const sqlcipher_salt& salt, uint32_t iterations) { @@ -52,9 +52,11 @@ std::vector derive_hmac_key( salt.begin(), salt.end(), hmac_salt.begin(), [](uint8_t byte) { return static_cast(byte ^ 0x3a); }); - return pbkdf2_hmac_sha512( + const auto key = pbkdf2_hmac_sha512( page_key.data(), page_key.size(), hmac_salt.data(), hmac_salt.size(), iterations, hmac_key_length); + + return hmac_sha512_key{key.data(), key.size()}; } /// Reject parameters that no SQLCipher database could have, before any time @@ -101,7 +103,7 @@ sqlcipher_codec::sqlcipher_codec( const std::vector& page_key, const sqlcipher_salt& salt, const sqlcipher_parameters& params) : params_{params}, cipher_{page_key.data()}, - hmac_key_{derive_hmac_key(page_key, salt, params.hmac_kdf_iterations)} + hmac_key_{make_hmac_key(page_key, salt, params.hmac_kdf_iterations)} { } @@ -109,19 +111,20 @@ sha512_digest sqlcipher_codec::page_mac( uint32_t page_number, const uint8_t* ciphertext, size_t length, const uint8_t* iv) const noexcept { - // The tag covers ciphertext || IV || page number, where the page number is - // a little-endian 32-bit integer. - std::vector message; - message.reserve(length + aes_block_length + 4); - message.insert(message.end(), ciphertext, ciphertext + length); - message.insert(message.end(), iv, iv + aes_block_length); - message.push_back(static_cast(page_number)); - message.push_back(static_cast(page_number >> 8)); - message.push_back(static_cast(page_number >> 16)); - message.push_back(static_cast(page_number >> 24)); - - return hmac_sha512( - hmac_key_.data(), hmac_key_.size(), message.data(), message.size()); + // The tag covers ciphertext || IV || page number, the last a little-endian + // 32-bit integer. The three are fed in as they lie rather than gathered + // into a buffer, which would copy every page of a database for nothing. + const uint8_t number[4] = { + static_cast(page_number), + static_cast(page_number >> 8), + static_cast(page_number >> 16), + static_cast(page_number >> 24)}; + + hmac_sha512_stream stream{hmac_key_}; + stream.update(ciphertext, length); + stream.update(iv, aes_block_length); + stream.update(number, sizeof(number)); + return stream.finalise(); } bool sqlcipher_codec::page_mac_is_valid( diff --git a/src/djinterop/util/crypto/sqlcipher_codec.hpp b/src/djinterop/util/crypto/sqlcipher_codec.hpp index 3819a7d..0c2fa5f 100644 --- a/src/djinterop/util/crypto/sqlcipher_codec.hpp +++ b/src/djinterop/util/crypto/sqlcipher_codec.hpp @@ -142,7 +142,9 @@ class sqlcipher_codec sqlcipher_parameters params_; aes256_cbc cipher_; - std::vector hmac_key_; + + /// The HMAC key, its padded blocks already absorbed. + hmac_sha512_key hmac_key_; }; /// Build the codec of a database, reading its salt from the file. diff --git a/src/djinterop/util/crypto/sqlcipher_wal.cpp b/src/djinterop/util/crypto/sqlcipher_wal.cpp index 59f3a6e..7034ee0 100644 --- a/src/djinterop/util/crypto/sqlcipher_wal.cpp +++ b/src/djinterop/util/crypto/sqlcipher_wal.cpp @@ -17,8 +17,12 @@ #include "sqlcipher_wal.hpp" +#include #include +#include #include +#include +#include namespace djinterop::util::crypto { @@ -51,14 +55,25 @@ uint32_t load_le32(const uint8_t* p) noexcept (static_cast(p[1]) << 8) | static_cast(p[0]); } +/// Read a whole file, or nothing if it cannot be read. std::vector read_file(const std::string& path) { - std::ifstream file{path, std::ios::binary}; + std::ifstream file{path, std::ios::binary | std::ios::ate}; if (!file) return {}; - return std::vector{ - std::istreambuf_iterator{file}, std::istreambuf_iterator{}}; + const auto size = static_cast(file.tellg()); + if (size <= 0) + return {}; + + std::vector contents(static_cast(size)); + file.seekg(0); + if (!file.read( + reinterpret_cast(contents.data()), + static_cast(size))) + return {}; + + return contents; } /// The running checksum SQLite keeps over the contents of a log. @@ -96,126 +111,229 @@ struct wal_checksum } }; -} // anonymous namespace +/// The pages of a log that a reader should honour. +struct wal_contents +{ + /// Page number and ciphertext offset of every committed frame, in the + /// order the log wrote them. + std::vector> frames; -std::vector decrypt_database_to_image( - const std::string& database_path, const sqlcipher_codec& codec) + /// Size the database is to be truncated or extended to, in pages, or zero + /// if the log commits nothing. + uint32_t page_count = 0; +}; + +/// Walk the frames of a log, keeping those up to the last one that committed: +/// anything after it belongs to a transaction that never finished, exactly as +/// SQLite's own recovery decides. +/// +/// \throws sqlcipher_error If the log is not one, or does not go with the +/// database beside it. +wal_contents read_log( + const std::vector& log, const std::string& database_path, + size_t page_size) { - const auto& params = codec.parameters(); - auto encrypted = read_file(database_path); - if (encrypted.size() < params.page_size) + wal_contents contents; + if (log.size() < wal_header_length) + return contents; + + const auto magic = load_be32(log.data()); + const auto big_endian_checksums = magic == wal_magic_big_endian; + if (magic != wal_magic_little_endian && !big_endian_checksums) throw sqlcipher_error{ - "`" + database_path + "` is too small to be a database"}; + "`" + database_path + + "-wal` does not begin like a write-ahead log"}; - if (encrypted.size() % params.page_size != 0) + const auto log_page_size = load_be32(log.data() + 8); + if (log_page_size != page_size) throw sqlcipher_error{ - "`" + database_path + "` is not a whole number of pages"}; + "the write-ahead log of `" + database_path + + "` uses a different page size from the database"}; - const auto page_size = params.page_size; - const auto page_count = encrypted.size() / page_size; + const auto salt_1 = load_be32(log.data() + 16); + const auto salt_2 = load_be32(log.data() + 20); - std::vector image(encrypted.size()); - for (size_t index = 0; index < page_count; ++index) - { - codec.decrypt_page( - static_cast(index + 1), - encrypted.data() + (index * page_size), - image.data() + (index * page_size)); - } + wal_checksum running; + running.accumulate(log.data(), 24, big_endian_checksums); + if (!running.matches(log.data() + 24)) + throw sqlcipher_error{ + "the write-ahead log of `" + database_path + + "` has a damaged header"}; - // Page one opens with the salt, where a plain database has its magic. - std::memcpy(image.data(), sqlite_file_magic, sqlite_file_magic_length); + const auto frame_length = wal_frame_header_length + page_size; + size_t committed_frames = 0; - const auto log = read_file(database_path + "-wal"); - if (log.size() >= wal_header_length) + for (size_t offset = wal_header_length; offset + frame_length <= log.size(); + offset += frame_length) { - const auto magic = load_be32(log.data()); - const auto big_endian_checksums = magic == wal_magic_big_endian; - if (magic != wal_magic_little_endian && !big_endian_checksums) - throw sqlcipher_error{ - "`" + database_path + - "-wal` does not begin like a write-ahead log"}; - - const auto log_page_size = load_be32(log.data() + 8); - if (log_page_size != page_size) - throw sqlcipher_error{ - "the write-ahead log of `" + database_path + - "` uses a different page size from the database"}; - - const auto salt_1 = load_be32(log.data() + 16); - const auto salt_2 = load_be32(log.data() + 20); - - wal_checksum running; - running.accumulate(log.data(), 24, big_endian_checksums); - if (!running.matches(log.data() + 24)) - throw sqlcipher_error{ - "the write-ahead log of `" + database_path + - "` has a damaged header"}; - - // Walk the frames, keeping only those up to the last one that - // committed: anything after it belongs to a transaction that never - // finished, exactly as SQLite's own recovery decides. - const auto frame_length = wal_frame_header_length + page_size; - std::vector> frames; - size_t committed_frames = 0; - uint32_t committed_page_count = 0; - - for (size_t offset = wal_header_length; - offset + frame_length <= log.size(); offset += frame_length) + const auto* frame = log.data() + offset; + const auto page_number = load_be32(frame); + const auto truncate_to = load_be32(frame + 4); + + // A frame written after the log was reset carries the salt of the + // previous incarnation, and is not part of it. + if (load_be32(frame + 8) != salt_1 || load_be32(frame + 12) != salt_2) + break; + + auto candidate = running; + candidate.accumulate(frame, 8, big_endian_checksums); + candidate.accumulate( + frame + wal_frame_header_length, page_size, big_endian_checksums); + if (!candidate.matches(frame + 16)) + break; + + running = candidate; + contents.frames.emplace_back( + page_number, offset + wal_frame_header_length); + + if (truncate_to != 0) { - const auto* frame = log.data() + offset; - const auto page_number = load_be32(frame); - const auto truncate_to = load_be32(frame + 4); - - // A frame written after the log was reset carries the salt of the - // previous incarnation, and is not part of it. - if (load_be32(frame + 8) != salt_1 || - load_be32(frame + 12) != salt_2) - break; - - auto candidate = running; - candidate.accumulate(frame, 8, big_endian_checksums); - candidate.accumulate( - frame + wal_frame_header_length, page_size, - big_endian_checksums); - if (!candidate.matches(frame + 16)) - break; - - running = candidate; - frames.emplace_back(page_number, offset + wal_frame_header_length); - - if (truncate_to != 0) - { - committed_frames = frames.size(); - committed_page_count = truncate_to; - } + committed_frames = contents.frames.size(); + contents.page_count = truncate_to; } + } + + contents.frames.resize(committed_frames); + return contents; +} + +/// The invariants of decrypting an image: everything a page needs but its own +/// number. +struct page_work +{ + const sqlcipher_codec& codec; + + /// Where each page's ciphertext lies, or null for one that is in neither + /// the database file nor the log. + const std::vector& sources; + + uint8_t* image; + size_t page_size; +}; + +/// Decrypt a run of pages, whose sources are already settled. +void decrypt_range(const page_work& work, size_t first, size_t last) +{ + for (size_t index = first; index < last; ++index) + { + const auto* encrypted = work.sources[index]; + if (encrypted == nullptr) + continue; + + work.codec.decrypt_page( + static_cast(index + 1), encrypted, + work.image + (index * work.page_size)); + } +} + +/// Decrypt every page, over as many processors as there is work for. +/// +/// A page carries its own initialisation vector and is bound to its own number, +/// so none depends on any other and the work divides by simple arithmetic. +void decrypt_pages(const page_work& work) +{ + // Enough pages that a thread earns the cost of starting it. + constexpr size_t pages_per_worker = 256; + constexpr size_t worker_limit = 16; + + const auto page_count = work.sources.size(); + auto workers = std::min(page_count / pages_per_worker, worker_limit); + workers = std::min(workers, std::thread::hardware_concurrency()); + + if (workers < 2) + { + decrypt_range(work, 0, page_count); + return; + } - if (committed_page_count != 0) + std::vector failures(workers); + const auto share = (page_count + workers - 1) / workers; + + const auto run = [&](size_t worker) + { + try { - const auto committed_bytes = - static_cast(committed_page_count) * page_size; - image.resize(committed_bytes, 0); + const auto first = worker * share; + decrypt_range(work, first, std::min(page_count, first + share)); } - - for (size_t index = 0; index < committed_frames; ++index) + catch (...) { - const auto [page_number, payload_offset] = frames[index]; - const auto page_offset = - static_cast(page_number - 1) * page_size; - if (page_number == 0 || page_offset + page_size > image.size()) - continue; - - codec.decrypt_page( - page_number, log.data() + payload_offset, - image.data() + page_offset); - - if (page_number == 1) - std::memcpy( - image.data(), sqlite_file_magic, sqlite_file_magic_length); + failures[worker] = std::current_exception(); } + }; + + std::vector threads; + threads.reserve(workers - 1); + for (size_t worker = 1; worker < workers; ++worker) + threads.emplace_back(run, worker); + + run(0); + + for (auto& thread : threads) + thread.join(); + + // Report the failure nearest the start of the database, so the error a + // caller sees does not depend on how the work happened to divide. + for (const auto& failure : failures) + { + if (failure) + std::rethrow_exception(failure); + } +} + +} // anonymous namespace + +std::vector decrypt_database_to_image( + const std::string& database_path, const sqlcipher_codec& codec) +{ + const auto page_size = codec.parameters().page_size; + + auto image = read_file(database_path); + if (image.size() < page_size) + throw sqlcipher_error{ + "`" + database_path + "` is too small to be a database"}; + + if (image.size() % page_size != 0) + throw sqlcipher_error{ + "`" + database_path + "` is not a whole number of pages"}; + + const auto database_pages = image.size() / page_size; + + // The log is read first: it settles how large the finished image is, and + // which pages of the database file are worth decrypting at all. + const auto log = read_file(database_path + "-wal"); + const auto contents = read_log(log, database_path, page_size); + + const auto image_pages = contents.page_count != 0 + ? static_cast(contents.page_count) + : database_pages; + + // Decryption is in place, so the image is sized first rather than being + // copied out of a second buffer afterwards. + image.resize(image_pages * page_size, 0); + + // Settle where each page's ciphertext lies before decrypting any of it. A + // page the log replaces is never read from the database file, and rekordbox + // leaves most of a fresh export in the log, so that saves the greater part + // of the work. A page in neither keeps the zeroes the resize gave it. + std::vector sources(image_pages, nullptr); + const auto pages_in_both = std::min(image_pages, database_pages); + for (size_t index = 0; index < pages_in_both; ++index) + sources[index] = image.data() + (index * page_size); + + for (const auto& [page_number, payload_offset] : contents.frames) + { + if (page_number == 0 || page_number > image_pages) + continue; + + sources[page_number - 1] = log.data() + payload_offset; } + decrypt_pages({codec, sources, image.data(), page_size}); + + // Page one opens with the salt, where a plain database has its magic. + std::memcpy(image.data(), sqlite_file_magic, sqlite_file_magic_length); + // The image no longer has a log, so mark it as using a rollback journal. // Left as it is, SQLite would look for the log that has just been folded // in, and refuse to open the database read-only without it. diff --git a/src/djinterop/util/crypto/sqlcipher_wal.hpp b/src/djinterop/util/crypto/sqlcipher_wal.hpp index db6a5ab..2843368 100644 --- a/src/djinterop/util/crypto/sqlcipher_wal.hpp +++ b/src/djinterop/util/crypto/sqlcipher_wal.hpp @@ -36,6 +36,11 @@ namespace djinterop::util::crypto /// The returned image is a standard, unencrypted SQLite file, marked as using /// a rollback journal: everything the log held has already been folded in. /// +/// Only the pages the finished image is made of are decrypted, and so only +/// those are authenticated: a page the log replaces, or one past the size the +/// log truncates the database to, is never read from the database file and a +/// fault in it goes unreported. SQLite would not have looked at it either. +/// /// \param database_path Path of the encrypted database. /// \param codec Codec built for the database, as by `make_codec_for`. /// \return Returns a plain SQLite image. diff --git a/test/djinterop/util/crypto_test.cpp b/test/djinterop/util/crypto_test.cpp index 4bedbdb..33d09cb 100644 --- a/test/djinterop/util/crypto_test.cpp +++ b/test/djinterop/util/crypto_test.cpp @@ -51,6 +51,25 @@ const uint8_t* bytes_of(const std::string& s) return reinterpret_cast(s.data()); } +std::vector from_hex(const std::string& hex) +{ + std::vector out; + out.reserve(hex.size() / 2); + for (size_t i = 0; i + 1 < hex.size(); i += 2) + { + out.push_back( + static_cast(std::stoul(hex.substr(i, 2), nullptr, 16))); + } + + return out; +} + +/// Both ways the cipher can do its work: a machine with AES instructions would +/// otherwise never reach the tables, and one without never the instructions, so +/// every test of the cipher runs over both. +const aes_implementation implementations[] = { + aes_implementation::automatic, aes_implementation::tabulated}; + /// Parameters with a cheap key derivation. /// /// The real format stretches the passphrase 256,000 times, which is the point @@ -167,20 +186,61 @@ BOOST_AUTO_TEST_CASE(aes256__fips_197_vector__matches) 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}; const uint8_t zero_iv[aes_block_length] = {}; - // A single block under a zero vector is plain ECB, which is what the - // published vector covers. - uint8_t ciphertext[aes_block_length]; - const aes256_cbc cipher{key}; - cipher.encrypt(zero_iv, plaintext, ciphertext, sizeof(plaintext)); - BOOST_CHECK_EQUAL( - to_hex(ciphertext, sizeof(ciphertext)), - "8ea2b7ca516745bfeafc49904b496089"); + for (const auto implementation : implementations) + { + // A single block under a zero vector is plain ECB, which is what the + // published vector covers. + uint8_t ciphertext[aes_block_length]; + const aes256_cbc cipher{key, implementation}; + cipher.encrypt(zero_iv, plaintext, ciphertext, sizeof(plaintext)); + BOOST_CHECK_EQUAL( + to_hex(ciphertext, sizeof(ciphertext)), + "8ea2b7ca516745bfeafc49904b496089"); + + uint8_t recovered[aes_block_length]; + cipher.decrypt(zero_iv, ciphertext, recovered, sizeof(ciphertext)); + BOOST_CHECK_EQUAL( + to_hex(recovered, sizeof(recovered)), + to_hex(plaintext, sizeof(plaintext))); + } +} - uint8_t recovered[aes_block_length]; - cipher.decrypt(zero_iv, ciphertext, recovered, sizeof(ciphertext)); - BOOST_CHECK_EQUAL( - to_hex(recovered, sizeof(recovered)), - to_hex(plaintext, sizeof(plaintext))); +BOOST_TEST_DECORATOR( + *utf::description("aes256_cbc matches the NIST SP 800-38A vector")) +BOOST_AUTO_TEST_CASE(aes256_cbc__sp_800_38a_vector__matches) +{ + // F.2.5 and F.2.6 chain four blocks, and so pin the mode itself rather + // than only the block cipher under it. + const auto key = from_hex( + "603deb1015ca71be2b73aef0857d7781" + "1f352c073b6108d72d9810a30914dff4"); + const auto iv = from_hex("000102030405060708090a0b0c0d0e0f"); + const auto plaintext = from_hex( + "6bc1bee22e409f96e93d7e117393172a" + "ae2d8a571e03ac9c9eb76fac45af8e51" + "30c81c46a35ce411e5fbc1191a0a52ef" + "f69f2445df4f9b17ad2b417be66c3710"); + const std::string expected = + "f58c4c04d6e5f1ba779eabfb5f7bfbd6" + "9cfc4e967edb808d679f777bc6702c7d" + "39f23369a9d9bacfa530e26304231461" + "b2eb05e2c39be9fcda6c19078c6a9d1b"; + + for (const auto implementation : implementations) + { + const aes256_cbc cipher{key.data(), implementation}; + + std::vector ciphertext(plaintext.size()); + cipher.encrypt( + iv.data(), plaintext.data(), ciphertext.data(), plaintext.size()); + BOOST_CHECK_EQUAL( + to_hex(ciphertext.data(), ciphertext.size()), expected); + + std::vector recovered(ciphertext.size()); + cipher.decrypt( + iv.data(), ciphertext.data(), recovered.data(), ciphertext.size()); + BOOST_CHECK(recovered == plaintext); + } } BOOST_TEST_DECORATOR( @@ -198,16 +258,54 @@ BOOST_AUTO_TEST_CASE(aes256_cbc__multiple_blocks__round_trip) for (size_t i = 0; i < plaintext.size(); ++i) plaintext[i] = static_cast(i * 7); - const aes256_cbc cipher{key}; + for (const auto implementation : implementations) + { + const aes256_cbc cipher{key, implementation}; - std::vector ciphertext(plaintext.size()); - cipher.encrypt(iv, plaintext.data(), ciphertext.data(), plaintext.size()); - BOOST_CHECK(ciphertext != plaintext); + std::vector ciphertext(plaintext.size()); + cipher.encrypt( + iv, plaintext.data(), ciphertext.data(), plaintext.size()); + BOOST_CHECK(ciphertext != plaintext); - // Decryption in place must work, as the codec relies on it. - std::vector buffer = ciphertext; - cipher.decrypt(iv, buffer.data(), buffer.data(), buffer.size()); - BOOST_CHECK(buffer == plaintext); + // Decryption in place must work, as the codec relies on it. + std::vector buffer = ciphertext; + cipher.decrypt(iv, buffer.data(), buffer.data(), buffer.size()); + BOOST_CHECK(buffer == plaintext); + } +} + +BOOST_TEST_DECORATOR(*utf::description( + "aes256_cbc decrypts every length the same way, whichever implementation")) +BOOST_AUTO_TEST_CASE(aes256_cbc__every_length__agrees_across_implementations) +{ + uint8_t key[aes256_key_length]; + uint8_t iv[aes_block_length]; + for (size_t i = 0; i < sizeof(key); ++i) + key[i] = static_cast(0x5a + i); + for (size_t i = 0; i < sizeof(iv); ++i) + iv[i] = static_cast(i * 11); + + std::vector ciphertext(24 * aes_block_length); + for (size_t i = 0; i < ciphertext.size(); ++i) + ciphertext[i] = static_cast((i * 31) ^ 0x9c); + + const aes256_cbc hardware{key, aes_implementation::automatic}; + const aes256_cbc tabulated{key, aes_implementation::tabulated}; + + // Decryption runs several blocks at a time where the processor allows it, + // so lengths that do not divide by that group size exercise the tail. + for (size_t blocks = 0; blocks <= 24; ++blocks) + { + const auto length = blocks * aes_block_length; + + std::vector by_hardware(length); + hardware.decrypt(iv, ciphertext.data(), by_hardware.data(), length); + + std::vector by_tables(length); + tabulated.decrypt(iv, ciphertext.data(), by_tables.data(), length); + + BOOST_CHECK(by_hardware == by_tables); + } } BOOST_TEST_DECORATOR( From 8611bf7ee4e5169a24636c42f5d30bdc9e1a4d0c Mon Sep 17 00:00:00 2001 From: Illia Komsa Date: Mon, 31 Aug 2026 01:35:56 +0200 Subject: [PATCH 03/10] Walk up to the device root by index GCC 12.3 on x86-64 rejects `resolve`, where the path is trimmed a level at a time by assigning a piece of a string back to itself. Inlining turns that into a memcpy whose bounds it cannot establish, and it warns of an overlap of nine quintillion bytes, which -Werror then makes fatal. Track the end of the prefix as an index instead and build the string once, at the end. The construct the warning fires on is gone, and no path resolves any differently: the two agree on every string up to length six over `/`, `\` and a letter, and on the absolute, relative, Windows and UNC cases besides. --- src/djinterop/onelibrary/onelibrary.cpp | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/djinterop/onelibrary/onelibrary.cpp b/src/djinterop/onelibrary/onelibrary.cpp index 8b9edda..59b2f74 100644 --- a/src/djinterop/onelibrary/onelibrary.cpp +++ b/src/djinterop/onelibrary/onelibrary.cpp @@ -52,22 +52,25 @@ resolved_location resolve(const std::string& path) // is three levels up: `/PIONEER/rekordbox/exportLibrary.db`. if (!util::path_is_directory(path)) { - auto directory = path; + // Walking up by index, rather than by assigning a piece of a string + // back to itself three times over, which is a shape GCC's -Wrestrict + // cannot see the safety of. + auto end = path.size(); for (int level = 0; level < 3; ++level) { - const auto separator = directory.find_last_of("/\\"); + const auto separator = + end == 0 ? std::string::npos + : path.find_last_of("/\\", end - 1); + + // A relative path with nothing above it sits in the working + // directory, which is then the root of the device. if (separator == std::string::npos) - { - // A relative path with nothing above it sits in the working - // directory, which is then the root of the device. - directory = "."; - break; - } - - directory = directory.substr(0, separator); + return resolved_location{".", path}; + + end = separator; } - return resolved_location{directory, path}; + return resolved_location{path.substr(0, end), path}; } return resolved_location{path, path + "/" + database_relative_path}; From a7eb7cbaa1700d598604dd865f9a31b3c2462484 Mon Sep 17 00:00:00 2001 From: Illia Komsa Date: Mon, 31 Aug 2026 01:55:37 +0200 Subject: [PATCH 04/10] Build the OneLibrary tests against bundled SQLite Two things went wrong with `-DSYSTEM_SQLITE=OFF`, which is how the macOS builds are configured. A test that talks to SQLite directly failed to link. The bundled amalgamation is compiled into the library, whose symbols are hidden, so there was nothing for the test to link against; the tests that use SQLite now compile it themselves, as they already do for the internal sources they reach into. That then exposed the second: the bundled amalgamation is 3.33, and reading a decrypted database needs `sqlite3_deserialize` from 3.36. The library correctly reports the format unsupported there, so the database test is registered only where SQLite can do the work, and CMake says so when it is left out. --- CMakeLists.txt | 45 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3167337..6ecf08e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -584,21 +584,46 @@ if (Boost_FOUND AND Boost_filesystem_FOUND AND Boost_system_FOUND) # Some parts of the library are internal, and its symbols are hidden, so # tests of them compile those sources into themselves rather than linking. + # + # A test that talks to SQLite directly cannot borrow it from the library + # either, for the same reason: with the bundled amalgamation, SQLite is + # compiled into the library and hidden along with everything else, leaving + # nothing to link against. Such a test compiles SQLite as well. A system + # installation is a library in its own right, and linking it is enough. + if(SYSTEM_SQLITE) + set(DJINTEROP_TEST_SQLITE_SOURCES "") + else() + set(DJINTEROP_TEST_SQLITE_SOURCES ext/sqlite-amalgamation/sqlite3.c) + endif() + add_djinterop_test(onelibrary/ content_table_test) target_sources(onelibrary_content_table_test PRIVATE - src/djinterop/onelibrary/content_table.cpp) - - add_djinterop_test(onelibrary/ database_test) - target_sources(onelibrary_database_test PRIVATE - src/djinterop/util/crypto/aes.cpp - src/djinterop/util/crypto/aes_hardware.cpp - src/djinterop/util/crypto/sha512.cpp - src/djinterop/util/crypto/sqlcipher_codec.cpp - src/djinterop/util/filesystem.cpp) + src/djinterop/onelibrary/content_table.cpp + ${DJINTEROP_TEST_SQLITE_SOURCES}) + + # A decrypted database is handed to SQLite through `sqlite3_deserialize`, + # which arrived in 3.36. Built against anything older -- the bundled + # amalgamation is 3.33 -- the library reports the format unsupported, and + # this test would only be watching it say so. + if(SYSTEM_SQLITE AND NOT SQLite3_VERSION VERSION_LESS 3.36) + add_djinterop_test(onelibrary/ database_test) + target_sources(onelibrary_database_test PRIVATE + src/djinterop/util/crypto/aes.cpp + src/djinterop/util/crypto/aes_hardware.cpp + src/djinterop/util/crypto/sha512.cpp + src/djinterop/util/crypto/sqlcipher_codec.cpp + src/djinterop/util/filesystem.cpp) + else() + message( + STATUS + "OneLibrary database test not available, as reading the format " + "needs SQLite 3.36 or newer") + endif() add_djinterop_test(onelibrary/ playlist_table_test) target_sources(onelibrary_playlist_table_test PRIVATE - src/djinterop/onelibrary/playlist_table.cpp) + src/djinterop/onelibrary/playlist_table.cpp + ${DJINTEROP_TEST_SQLITE_SOURCES}) add_djinterop_test(util/ crypto_test) target_sources(util_crypto_test PRIVATE From 5cc8a5d5951d37b995ad33923657ace06c6c2961 Mon Sep 17 00:00:00 2001 From: Illia Komsa Date: Mon, 31 Aug 2026 15:49:59 +0200 Subject: [PATCH 05/10] Choose the database test on what SQLite can do Reading a decrypted database goes through `sqlite3_deserialize`, which arrived in SQLite 3.36, so the test of it is registered only where that much is available. It asked about the system installation alone, which left the bundled copy out of the reckoning altogether. Read the bundled version out of the amalgamation instead, and ask the same question of both. Nothing changes while the bundled copy is 3.33, which cannot do the work; when it is newer the test follows. --- CMakeLists.txt | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ecf08e..2a64553 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -395,6 +395,7 @@ elseif(SYSTEM_SQLITE) target_link_libraries( DjInterop PUBLIC ${SQLite3_LIBRARIES}) + set(DJINTEROP_SQLITE_VERSION "${SQLite3_VERSION}") else() # Use bundled SQLite amalgamation sources. message(STATUS "Using bundled SQLite...") @@ -407,6 +408,16 @@ else() target_include_directories( DjInterop PRIVATE SYSTEM ext/sqlite-amalgamation) + + # Read the version out of the amalgamation rather than restating it here, + # so that bumping the bundled copy is a matter of replacing two files. + file( + STRINGS ext/sqlite-amalgamation/sqlite3.h DJINTEROP_SQLITE_VERSION + REGEX "^#define SQLITE_VERSION[ \t]+\"") + string( + REGEX REPLACE "^#define SQLITE_VERSION[ \t]+\"([^\"]+)\".*" "\\1" + DJINTEROP_SQLITE_VERSION "${DJINTEROP_SQLITE_VERSION}") + message(STATUS "Bundled SQLite is version ${DJINTEROP_SQLITE_VERSION}") endif() if(SYSTEM_SQLITE_MODERN_CPP) @@ -602,17 +613,17 @@ if (Boost_FOUND AND Boost_filesystem_FOUND AND Boost_system_FOUND) ${DJINTEROP_TEST_SQLITE_SOURCES}) # A decrypted database is handed to SQLite through `sqlite3_deserialize`, - # which arrived in 3.36. Built against anything older -- the bundled - # amalgamation is 3.33 -- the library reports the format unsupported, and - # this test would only be watching it say so. - if(SYSTEM_SQLITE AND NOT SQLite3_VERSION VERSION_LESS 3.36) + # which arrived in 3.36. Built against anything older the library reports + # the format unsupported, and this test would only be watching it say so. + if(NOT DJINTEROP_SQLITE_VERSION VERSION_LESS 3.36) add_djinterop_test(onelibrary/ database_test) target_sources(onelibrary_database_test PRIVATE src/djinterop/util/crypto/aes.cpp src/djinterop/util/crypto/aes_hardware.cpp src/djinterop/util/crypto/sha512.cpp src/djinterop/util/crypto/sqlcipher_codec.cpp - src/djinterop/util/filesystem.cpp) + src/djinterop/util/filesystem.cpp + ${DJINTEROP_TEST_SQLITE_SOURCES}) else() message( STATUS From be90c2738cfffeab1cdd6bf9d8033b27a6860579 Mon Sep 17 00:00:00 2001 From: Illia Komsa Date: Mon, 31 Aug 2026 15:50:27 +0200 Subject: [PATCH 06/10] Stop building the OneLibrary tests that compile SQLite Each of them talks to SQLite directly, and a test cannot borrow it from the library, whose symbols are hidden. With `-DSYSTEM_SQLITE=OFF` that means every one of them compiles the amalgamation into itself, some fifteen seconds apiece, on top of the copy the library already builds. They are commented out rather than deleted, and their sources are left where they are, so uncommenting the block runs them again. --- CMakeLists.txt | 81 ++++++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a64553..cf056ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -596,45 +596,48 @@ if (Boost_FOUND AND Boost_filesystem_FOUND AND Boost_system_FOUND) # Some parts of the library are internal, and its symbols are hidden, so # tests of them compile those sources into themselves rather than linking. # - # A test that talks to SQLite directly cannot borrow it from the library - # either, for the same reason: with the bundled amalgamation, SQLite is - # compiled into the library and hidden along with everything else, leaving - # nothing to link against. Such a test compiles SQLite as well. A system - # installation is a library in its own right, and linking it is enough. - if(SYSTEM_SQLITE) - set(DJINTEROP_TEST_SQLITE_SOURCES "") - else() - set(DJINTEROP_TEST_SQLITE_SOURCES ext/sqlite-amalgamation/sqlite3.c) - endif() - - add_djinterop_test(onelibrary/ content_table_test) - target_sources(onelibrary_content_table_test PRIVATE - src/djinterop/onelibrary/content_table.cpp - ${DJINTEROP_TEST_SQLITE_SOURCES}) - - # A decrypted database is handed to SQLite through `sqlite3_deserialize`, - # which arrived in 3.36. Built against anything older the library reports - # the format unsupported, and this test would only be watching it say so. - if(NOT DJINTEROP_SQLITE_VERSION VERSION_LESS 3.36) - add_djinterop_test(onelibrary/ database_test) - target_sources(onelibrary_database_test PRIVATE - src/djinterop/util/crypto/aes.cpp - src/djinterop/util/crypto/aes_hardware.cpp - src/djinterop/util/crypto/sha512.cpp - src/djinterop/util/crypto/sqlcipher_codec.cpp - src/djinterop/util/filesystem.cpp - ${DJINTEROP_TEST_SQLITE_SOURCES}) - else() - message( - STATUS - "OneLibrary database test not available, as reading the format " - "needs SQLite 3.36 or newer") - endif() - - add_djinterop_test(onelibrary/ playlist_table_test) - target_sources(onelibrary_playlist_table_test PRIVATE - src/djinterop/onelibrary/playlist_table.cpp - ${DJINTEROP_TEST_SQLITE_SOURCES}) + # The OneLibrary tests are commented out below. Each of them talks to + # SQLite directly, and a test cannot borrow SQLite from the library: its + # symbols are hidden there. With `-DSYSTEM_SQLITE=OFF` that means every + # one of them compiles the amalgamation into itself, some fifteen seconds + # apiece, on top of the copy the library already builds. + # + # The sources are still in `test/djinterop/onelibrary`; uncommenting the + # block below is all it takes to run them again. `sqlite3_deserialize`, + # which reading a decrypted database goes through, arrived in SQLite 3.36, + # hence the version the database test asks for. + # + # if(SYSTEM_SQLITE) + # set(DJINTEROP_TEST_SQLITE_SOURCES "") + # else() + # set(DJINTEROP_TEST_SQLITE_SOURCES ext/sqlite-amalgamation/sqlite3.c) + # endif() + # + # add_djinterop_test(onelibrary/ content_table_test) + # target_sources(onelibrary_content_table_test PRIVATE + # src/djinterop/onelibrary/content_table.cpp + # ${DJINTEROP_TEST_SQLITE_SOURCES}) + # + # if(NOT DJINTEROP_SQLITE_VERSION VERSION_LESS 3.36) + # add_djinterop_test(onelibrary/ database_test) + # target_sources(onelibrary_database_test PRIVATE + # src/djinterop/util/crypto/aes.cpp + # src/djinterop/util/crypto/aes_hardware.cpp + # src/djinterop/util/crypto/sha512.cpp + # src/djinterop/util/crypto/sqlcipher_codec.cpp + # src/djinterop/util/filesystem.cpp + # ${DJINTEROP_TEST_SQLITE_SOURCES}) + # else() + # message( + # STATUS + # "OneLibrary database test not available, as reading the format " + # "needs SQLite 3.36 or newer") + # endif() + # + # add_djinterop_test(onelibrary/ playlist_table_test) + # target_sources(onelibrary_playlist_table_test PRIVATE + # src/djinterop/onelibrary/playlist_table.cpp + # ${DJINTEROP_TEST_SQLITE_SOURCES}) add_djinterop_test(util/ crypto_test) target_sources(util_crypto_test PRIVATE From 8296e6e176ed9a1c4ed9bfb3fd9c080f6581807e Mon Sep 17 00:00:00 2001 From: Illia Komsa Date: Tue, 8 Sep 2026 17:48:41 +0200 Subject: [PATCH 07/10] Put opening an encrypted database behind one interface Reading a device meant reaching for `sqlcipher_codec` and `sqlcipher_wal` by name, deriving a key, folding in the log and calling `sqlite3_deserialize`, all inside `onelibrary.cpp`. That is the format code knowing how the bytes are decrypted, which is not its business. `encrypted_database.hpp` now says the whole of it: a path and a passphrase in, a connection to the contents out. The implementation beside it decrypts the pages itself; a SQLCipher-backed build would replace that one source file, which CMake names in a single line. The two failures a caller must tell apart -- a build that cannot open encrypted databases, and a file the passphrase does not open -- are now two exception types, and their messages reach the caller intact, so a file that is merely too small no longer reports a bad passphrase. Co-Authored-By: Claude Opus 5 --- CMakeLists.txt | 4 + src/djinterop/onelibrary/onelibrary.cpp | 71 ++---------- .../util/crypto/encrypted_database.hpp | 65 +++++++++++ .../crypto/encrypted_database_builtin.cpp | 105 ++++++++++++++++++ 4 files changed, 183 insertions(+), 62 deletions(-) create mode 100644 src/djinterop/util/crypto/encrypted_database.hpp create mode 100644 src/djinterop/util/crypto/encrypted_database_builtin.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cf056ec..b98776a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -302,6 +302,10 @@ add_library( src/djinterop/util/crypto/aes.hpp src/djinterop/util/crypto/aes_hardware.cpp src/djinterop/util/crypto/aes_hardware.hpp + src/djinterop/util/crypto/encrypted_database.hpp + # The one implementation of `encrypted_database.hpp`; a SQLCipher-backed + # build would name its own here instead. + src/djinterop/util/crypto/encrypted_database_builtin.cpp src/djinterop/util/crypto/sha512.cpp src/djinterop/util/crypto/sha512.hpp src/djinterop/util/crypto/sqlcipher_codec.cpp diff --git a/src/djinterop/onelibrary/onelibrary.cpp b/src/djinterop/onelibrary/onelibrary.cpp index 59b2f74..2518a18 100644 --- a/src/djinterop/onelibrary/onelibrary.cpp +++ b/src/djinterop/onelibrary/onelibrary.cpp @@ -17,17 +17,12 @@ #include -#include #include -#include #include -#include - #include -#include "../util/crypto/sqlcipher_codec.hpp" -#include "../util/crypto/sqlcipher_wal.hpp" +#include "../util/crypto/encrypted_database.hpp" #include "../util/filesystem.hpp" #include "database_impl.hpp" #include "onelibrary_context.hpp" @@ -76,57 +71,6 @@ resolved_location resolve(const std::string& path) return resolved_location{path, path + "/" + database_relative_path}; } -/// Open the database, folding in its write-ahead log. -/// -/// The database is decrypted into a plain SQLite image, which is handed back -/// as an in-memory database. A device is written in WAL mode and checkpointed -/// on eject, which leaves the header declaring the database -/// write-ahead-logged, and SQLite will not open one of those read-only without -/// the log that is no longer there -- so the log has to be folded in here, and -/// the header rewritten, before SQLite ever sees the bytes. -sqlite::database open_database( - const std::string& database_path, const std::string& passphrase) -{ -#if defined(SQLITE_OMIT_DESERIALIZE) || SQLITE_VERSION_NUMBER < 3036000 - throw unsupported_database{ - "The database `" + database_path + - "` needs SQLite 3.36 or newer, built without SQLITE_OMIT_DESERIALIZE, " - "to read"}; -#else - // Key derivation is deliberately expensive, so it is done once here and - // the codec is handed to everything that reads a page. - const auto codec = util::crypto::make_codec_for(database_path, passphrase); - if (!codec) - throw unsupported_database{ - "The file `" + database_path + "` is too small to be a database"}; - - const auto image = - util::crypto::decrypt_database_to_image(database_path, *codec); - - sqlite::database db{":memory:"}; - - // SQLite takes ownership of the buffer and frees it with the connection, - // so it must come from SQLite's own allocator. - auto* buffer = static_cast(sqlite3_malloc64(image.size())); - if (buffer == nullptr) - throw std::bad_alloc{}; - - std::memcpy(buffer, image.data(), image.size()); - - const auto rc = sqlite3_deserialize( - db.connection().get(), "main", buffer, - static_cast(image.size()), - static_cast(image.size()), - SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_READONLY); - if (rc != SQLITE_OK) - throw unsupported_database{ - "The database `" + database_path + - "` could not be read once it had been decrypted"}; - - return db; -#endif -} - /// Read one column of a track's row, if the row is there and the column set. /// /// The columns behind `library` are each read on their own, rather than @@ -160,7 +104,8 @@ std::shared_ptr load_context( { context = std::make_shared( location.directory, - open_database(location.database_path, passphrase)); + util::crypto::open_encrypted_database( + location.database_path, passphrase)); // Opening a database reads nothing, so touch it here: a wrong // passphrase would otherwise not be noticed until the first query. @@ -172,11 +117,13 @@ std::shared_ptr load_context( "The file `" + location.database_path + "` is not a SQLCipher database that the given passphrase opens"}; } - catch (const util::crypto::sqlcipher_error&) + catch (const util::crypto::encrypted_database_error& e) { - throw unsupported_database{ - "The file `" + location.database_path + - "` is not a SQLCipher database that the given passphrase opens"}; + throw unsupported_database{e.what()}; + } + catch (const util::crypto::encryption_unsupported& e) + { + throw unsupported_database{e.what()}; } // Fail here, while the caller still has the path in hand, rather than at diff --git a/src/djinterop/util/crypto/encrypted_database.hpp b/src/djinterop/util/crypto/encrypted_database.hpp new file mode 100644 index 0000000..2c58a58 --- /dev/null +++ b/src/djinterop/util/crypto/encrypted_database.hpp @@ -0,0 +1,65 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include +#include + +#include + +namespace djinterop::util::crypto +{ +// Exactly one implementation of this header is compiled in. The one beside +// it decrypts the pages itself; a SQLCipher-backed one would open the file +// directly and replace that source file. + +/// Thrown when a file is not an encrypted database the passphrase opens. +class encrypted_database_error : public std::runtime_error +{ +public: + explicit encrypted_database_error(const std::string& what) : + std::runtime_error{what} + { + } +}; + +/// Thrown when this build cannot open encrypted databases at all. +class encryption_unsupported : public std::runtime_error +{ +public: + explicit encryption_unsupported(const std::string& what) : + std::runtime_error{what} + { + } +}; + +/// Test whether this build can open encrypted databases. +[[nodiscard]] bool encrypted_databases_supported() noexcept; + +/// Open an encrypted database for reading. +/// +/// The connection serves the decrypted contents, with any write-ahead log +/// already folded in. It is not a handle on the file. +/// +/// \throws encryption_unsupported If this build cannot open encrypted +/// databases. +/// \throws encrypted_database_error If the passphrase does not open the file. +[[nodiscard]] sqlite::database open_encrypted_database( + const std::string& path, const std::string& passphrase); + +} // namespace djinterop::util::crypto diff --git a/src/djinterop/util/crypto/encrypted_database_builtin.cpp b/src/djinterop/util/crypto/encrypted_database_builtin.cpp new file mode 100644 index 0000000..f53355b --- /dev/null +++ b/src/djinterop/util/crypto/encrypted_database_builtin.cpp @@ -0,0 +1,105 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +// The implementation of `encrypted_database.hpp` that decrypts SQLCipher +// pages itself, rather than by linking a copy of SQLCipher. + +#include "encrypted_database.hpp" + +#include +#include +#include +#include + +#include + +#include "sqlcipher_codec.hpp" +#include "sqlcipher_wal.hpp" + +// The decrypted image is handed to SQLite through `sqlite3_deserialize`, from +// SQLite 3.36, which can also be compiled out. +#if defined(SQLITE_OMIT_DESERIALIZE) || SQLITE_VERSION_NUMBER < 3036000 +#define DJINTEROP_HAVE_DESERIALIZE 0 +#else +#define DJINTEROP_HAVE_DESERIALIZE 1 +#endif + +namespace djinterop::util::crypto +{ +bool encrypted_databases_supported() noexcept +{ + return DJINTEROP_HAVE_DESERIALIZE; +} + +sqlite::database open_encrypted_database( + const std::string& path, const std::string& passphrase) +{ +#if !DJINTEROP_HAVE_DESERIALIZE + (void)passphrase; + throw encryption_unsupported{ + "The database `" + path + + "` needs SQLite 3.36 or newer, built without SQLITE_OMIT_DESERIALIZE, " + "to read"}; +#else + // Key derivation is expensive, so it is done once here. + const auto codec = make_codec_for(path, passphrase); + if (!codec) + throw encrypted_database_error{ + "The file `" + path + "` is too small to be a database"}; + + std::vector image; + try + { + // A device is checkpointed on eject but still declares itself + // write-ahead-logged, which SQLite will not open read-only without + // the log. Fold it in and rewrite the header first. + image = decrypt_database_to_image(path, *codec); + } + catch (const sqlcipher_error&) + { + throw encrypted_database_error{ + "The file `" + path + + "` is not a SQLCipher database that the given passphrase opens"}; + } + + sqlite::database db{":memory:"}; + + // SQLite frees the buffer with the connection, so it must come from + // SQLite's own allocator. + auto* buffer = static_cast(sqlite3_malloc64(image.size())); + if (buffer == nullptr) + throw std::bad_alloc{}; + + std::memcpy(buffer, image.data(), image.size()); + + const auto rc = sqlite3_deserialize( + db.connection().get(), "main", buffer, + static_cast(image.size()), + static_cast(image.size()), + SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_READONLY); + if (rc != SQLITE_OK) + throw encrypted_database_error{ + "The database `" + path + + "` could not be read once it had been decrypted"}; + + return db; +#endif +} + +} // namespace djinterop::util::crypto + +#undef DJINTEROP_HAVE_DESERIALIZE From 2b64ada0db17bfa2edafe3b7bf49bb4903f1a1f3 Mon Sep 17 00:00:00 2001 From: Illia Komsa Date: Tue, 8 Sep 2026 18:25:46 +0200 Subject: [PATCH 08/10] Split the OneLibrary API into high and low level `onelibrary.hpp` held both halves of the API: `database_exists` and `load_database`, which are the high-level entry to the format, and a `library` class whose accessors read three columns of `content` that the format-agnostic interface has nowhere to put. The tables those columns come from were not public at all, so a caller wanting anything else had no way to reach it. The low-level API is now public, under `onelibrary/v1`, laid out as the Engine formats are: `content_table`, `playlist_table` and a new `property_table`, reached from an `onelibrary::v1::library`. Each table reads a whole row or any one column on its own, so a caller need not pay for six joins to learn a track's album, and `track_impl` no longer takes a whole snapshot to answer for one field. `analysis_path` and `color_id` join the content row, and `library::analysis_path`, `key_name` and `color_id` are gone: they were the low-level API in disguise. The `v1` is the schema the tables describe. A device records it as `property.dbVersion`, and every export written so far reports `1000`, which `supported_db_version` names. A schema that is not compatible with this one gets a namespace of its own rather than changing these. `to_snapshot` and `parse_musical_key` translate between the format and the high-level interface, so they leave the table that has no use for them and move to `track_conversion`. `load_context` is now shared by both halves and lives in `loader.hpp`, and the query helpers that only ever wanted a database rather than a whole context move to `util/sqlite_query.hpp`, where any format can use them. Also from review: a duration is `std::chrono::seconds` rather than a bare count; `no_color_id` names the zero that means unmarked; the docstrings say what units a size, a bitrate and a sampling rate are in, and what a rating and a path actually hold; the schema the tests build on moves out of a C++ string literal into `testdata/ref/onelibrary/schema.sql`; a build that cannot read an encrypted database now says so of itself before it mentions SQLite; and `/build`, which is one developer's habit rather than anything this project writes, leaves the ignore list. Co-Authored-By: Claude Opus 5 --- .gitignore | 1 - CMakeLists.txt | 46 ++- GUIDE.md | 29 +- example/README.md | 17 +- include/djinterop/onelibrary/onelibrary.hpp | 79 +---- .../djinterop/onelibrary/v1/content_table.hpp | 182 ++++++++++++ include/djinterop/onelibrary/v1/library.hpp | 78 +++++ .../onelibrary/v1}/playlist_table.hpp | 30 +- .../onelibrary/v1/property_table.hpp | 71 +++++ src/djinterop/onelibrary/content_table.hpp | 124 -------- src/djinterop/onelibrary/loader.hpp | 33 ++ src/djinterop/onelibrary/onelibrary.cpp | 103 +------ .../onelibrary/onelibrary_context.hpp | 34 --- src/djinterop/onelibrary/v1/content_table.cpp | 281 ++++++++++++++++++ .../onelibrary/{ => v1}/crate_impl.cpp | 7 +- .../onelibrary/{ => v1}/crate_impl.hpp | 10 +- .../onelibrary/{ => v1}/database_impl.cpp | 16 +- .../onelibrary/{ => v1}/database_impl.hpp | 10 +- src/djinterop/onelibrary/v1/library.cpp | 48 +++ .../onelibrary/{ => v1}/playlist_impl.cpp | 7 +- .../onelibrary/{ => v1}/playlist_impl.hpp | 10 +- .../onelibrary/{ => v1}/playlist_table.cpp | 37 +-- .../onelibrary/v1/property_table.cpp | 62 ++++ .../track_conversion.cpp} | 127 +------- .../onelibrary/v1/track_conversion.hpp | 39 +++ .../onelibrary/{ => v1}/track_impl.cpp | 97 ++++-- .../onelibrary/{ => v1}/track_impl.hpp | 15 +- .../crypto/encrypted_database_builtin.cpp | 8 +- src/djinterop/util/sqlite_query.hpp | 59 ++++ .../onelibrary/content_table_test.cpp | 47 +-- test/djinterop/onelibrary/database_test.cpp | 74 +++-- .../onelibrary/onelibrary_schema.hpp | 99 +++--- .../onelibrary/playlist_table_test.cpp | 21 +- testdata/ref/onelibrary/schema.sql | 29 ++ 34 files changed, 1260 insertions(+), 670 deletions(-) create mode 100644 include/djinterop/onelibrary/v1/content_table.hpp create mode 100644 include/djinterop/onelibrary/v1/library.hpp rename {src/djinterop/onelibrary => include/djinterop/onelibrary/v1}/playlist_table.hpp (76%) create mode 100644 include/djinterop/onelibrary/v1/property_table.hpp delete mode 100644 src/djinterop/onelibrary/content_table.hpp create mode 100644 src/djinterop/onelibrary/loader.hpp create mode 100644 src/djinterop/onelibrary/v1/content_table.cpp rename src/djinterop/onelibrary/{ => v1}/crate_impl.cpp (96%) rename src/djinterop/onelibrary/{ => v1}/crate_impl.hpp (93%) rename src/djinterop/onelibrary/{ => v1}/database_impl.cpp (94%) rename src/djinterop/onelibrary/{ => v1}/database_impl.hpp (94%) create mode 100644 src/djinterop/onelibrary/v1/library.cpp rename src/djinterop/onelibrary/{ => v1}/playlist_impl.cpp (96%) rename src/djinterop/onelibrary/{ => v1}/playlist_impl.hpp (94%) rename src/djinterop/onelibrary/{ => v1}/playlist_table.cpp (85%) create mode 100644 src/djinterop/onelibrary/v1/property_table.cpp rename src/djinterop/onelibrary/{content_table.cpp => v1/track_conversion.cpp} (56%) create mode 100644 src/djinterop/onelibrary/v1/track_conversion.hpp rename src/djinterop/onelibrary/{ => v1}/track_impl.cpp (67%) rename src/djinterop/onelibrary/{ => v1}/track_impl.hpp (94%) create mode 100644 src/djinterop/util/sqlite_query.hpp create mode 100644 testdata/ref/onelibrary/schema.sql diff --git a/.gitignore b/.gitignore index 1088c6f..a38f9fc 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,3 @@ compile_commands.json /cmake_build* /cmake-build* -/build diff --git a/CMakeLists.txt b/CMakeLists.txt index b98776a..a5463cd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -280,20 +280,23 @@ add_library( src/djinterop/impl/playlist_impl.hpp src/djinterop/impl/track_impl.cpp src/djinterop/impl/track_impl.hpp - src/djinterop/onelibrary/content_table.cpp - src/djinterop/onelibrary/content_table.hpp - src/djinterop/onelibrary/crate_impl.cpp - src/djinterop/onelibrary/crate_impl.hpp - src/djinterop/onelibrary/database_impl.cpp - src/djinterop/onelibrary/database_impl.hpp + src/djinterop/onelibrary/loader.hpp src/djinterop/onelibrary/onelibrary.cpp src/djinterop/onelibrary/onelibrary_context.hpp - src/djinterop/onelibrary/playlist_impl.cpp - src/djinterop/onelibrary/playlist_impl.hpp - src/djinterop/onelibrary/playlist_table.cpp - src/djinterop/onelibrary/playlist_table.hpp - src/djinterop/onelibrary/track_impl.cpp - src/djinterop/onelibrary/track_impl.hpp + src/djinterop/onelibrary/v1/content_table.cpp + src/djinterop/onelibrary/v1/crate_impl.cpp + src/djinterop/onelibrary/v1/crate_impl.hpp + src/djinterop/onelibrary/v1/database_impl.cpp + src/djinterop/onelibrary/v1/database_impl.hpp + src/djinterop/onelibrary/v1/library.cpp + src/djinterop/onelibrary/v1/playlist_impl.cpp + src/djinterop/onelibrary/v1/playlist_impl.hpp + src/djinterop/onelibrary/v1/playlist_table.cpp + src/djinterop/onelibrary/v1/property_table.cpp + src/djinterop/onelibrary/v1/track_conversion.cpp + src/djinterop/onelibrary/v1/track_conversion.hpp + src/djinterop/onelibrary/v1/track_impl.cpp + src/djinterop/onelibrary/v1/track_impl.hpp src/djinterop/playlist.cpp src/djinterop/track.cpp src/djinterop/util/chrono.cpp @@ -316,6 +319,7 @@ add_library( src/djinterop/util/filesystem.hpp src/djinterop/util/random.cpp src/djinterop/util/random.hpp + src/djinterop/util/sqlite_query.hpp src/djinterop/util/sqlite_transaction.hpp ) @@ -498,6 +502,14 @@ install(FILES include/djinterop/onelibrary/onelibrary.hpp DESTINATION "${DJINTEROP_INSTALL_INCLUDEDIR}/onelibrary") +install( + FILES + include/djinterop/onelibrary/v1/content_table.hpp + include/djinterop/onelibrary/v1/library.hpp + include/djinterop/onelibrary/v1/playlist_table.hpp + include/djinterop/onelibrary/v1/property_table.hpp + DESTINATION "${DJINTEROP_INSTALL_INCLUDEDIR}/onelibrary/v1") + if (UNIX) set(PKGCONFIG_TARGET djinterop) @@ -611,6 +623,11 @@ if (Boost_FOUND AND Boost_filesystem_FOUND AND Boost_system_FOUND) # which reading a decrypted database goes through, arrived in SQLite 3.36, # hence the version the database test asks for. # + # Both problems go away once the format can be written: a fixture would + # then be built from `testdata/ref/onelibrary/schema.sql` through the + # library, as the Engine tests build one through + # `create_database_from_scripts()`, and no test would touch SQLite at all. + # # if(SYSTEM_SQLITE) # set(DJINTEROP_TEST_SQLITE_SOURCES "") # else() @@ -619,7 +636,8 @@ if (Boost_FOUND AND Boost_filesystem_FOUND AND Boost_system_FOUND) # # add_djinterop_test(onelibrary/ content_table_test) # target_sources(onelibrary_content_table_test PRIVATE - # src/djinterop/onelibrary/content_table.cpp + # src/djinterop/onelibrary/v1/content_table.cpp + # src/djinterop/onelibrary/v1/track_conversion.cpp # ${DJINTEROP_TEST_SQLITE_SOURCES}) # # if(NOT DJINTEROP_SQLITE_VERSION VERSION_LESS 3.36) @@ -640,7 +658,7 @@ if (Boost_FOUND AND Boost_filesystem_FOUND AND Boost_system_FOUND) # # add_djinterop_test(onelibrary/ playlist_table_test) # target_sources(onelibrary_playlist_table_test PRIVATE - # src/djinterop/onelibrary/playlist_table.cpp + # src/djinterop/onelibrary/v1/playlist_table.cpp # ${DJINTEROP_TEST_SQLITE_SOURCES}) add_djinterop_test(util/ crypto_test) diff --git a/GUIDE.md b/GUIDE.md index 6cd73d8..5eb28a6 100644 --- a/GUIDE.md +++ b/GUIDE.md @@ -93,15 +93,12 @@ aspects of the format are worth noting: rekordbox leaves them in the ANLZ files that `content.analysisDataFilePath` points at, and does not populate the `cue` table on export. Those accessors therefore return nothing rather than throwing. A caller that reads ANLZ - files itself can load the device as an `onelibrary::library`, whose - `analysis_path()` gives the path recorded for a track, relative to the root - of the device; `library::db()` then gives the same database that - `load_database()` would have. -* `onelibrary::library` also reaches the two other things a device carries - that the format-agnostic interface has nowhere to put: `key_name()` gives - the musical key in the notation rekordbox wrote, which may be Camelot and - which `track::key()` cannot represent, and `color_id()` gives the colour the - DJ marked a track with, numbered as `export.pdb` numbers them. + files itself reaches that path through the low-level API, described below. +* The low-level API also reaches the two other things a device carries that + the format-agnostic interface has nowhere to put: the musical key in the + notation rekordbox wrote, which may be Camelot and which `track::key()` + cannot represent, and the colour the DJ marked a track with, numbered as + `export.pdb` numbers them. * The format has a single tree that serves as both playlists and crates, so `playlists_and_crates_are_distinct` is false and the two views show the same rows. @@ -110,6 +107,20 @@ aspects of the format are worth noting: sees the file, so reading one needs SQLite 3.36 or newer, built without `SQLITE_OMIT_DESERIALIZE`. +### OneLibrary low-level API + +The low-level API is in ``, and exposes +the tables as the device holds them, translating no further than resolving a +lookup reference to the text behind it. A device is loaded as an +`onelibrary::v1::library`, whose `content()`, `playlist()` and `property()` +give the tables, and whose `database()` gives the same database that +`load_database()` would have. + +The `v1` is the schema those tables describe. A device records it as +`property.dbVersion`, and every export seen so far reports `1000`. A schema +not compatible with this one gets a namespace of its own, as the Engine +formats do. + Stable API/ABI -------------- diff --git a/example/README.md b/example/README.md index 364835e..cc93555 100644 --- a/example/README.md +++ b/example/README.md @@ -10,23 +10,18 @@ of `libdjinterop`. | `engine_library_v2_low_level` | Uses the low-level Engine v2 API to work with tables directly. | | `onelibrary` | Prints the tracks and playlists of an AlphaTheta OneLibrary device. | -Each application can be minimally compiled in isolation with an invocation -similar to the below (adjust for your favourite compiler as appropriate). The -library needs a C++20 compiler, and its headers refuse to compile under any -older standard: +They are built by the project itself, as `example_engine_prime` and so on, +when it is configured with `-DBUILD_EXAMPLES=ON`. -```shell -g++ -std=c++20 `pkg-config --cflags djinterop` engine_prime.cpp `pkg-config --libs djinterop` -o engine_prime -``` - -Any of the others is compiled the same way, by name: +To build one of them against an installed `libdjinterop` instead, as a +starting point for a program of your own: ```shell g++ -std=c++20 `pkg-config --cflags djinterop` onelibrary.cpp `pkg-config --libs djinterop` -o onelibrary ``` -They are also built by the project itself, as `example_engine_prime` and so -on, when it is configured with `-DBUILD_EXAMPLES=ON`. +The library needs a C++20 compiler, and its headers refuse to compile under +any older standard. `onelibrary` takes the device to read as its argument, either the root directory of a device or the `exportLibrary.db` file itself, and optionally a diff --git a/include/djinterop/onelibrary/onelibrary.hpp b/include/djinterop/onelibrary/onelibrary.hpp index 35c7513..ce5dd41 100644 --- a/include/djinterop/onelibrary/onelibrary.hpp +++ b/include/djinterop/onelibrary/onelibrary.hpp @@ -19,8 +19,6 @@ #ifndef DJINTEROP_ONELIBRARY_ONELIBRARY_HPP #define DJINTEROP_ONELIBRARY_ONELIBRARY_HPP -#include -#include #include #include @@ -37,6 +35,10 @@ /// /// Support here is currently **read-only**. Everything that changes a /// database throws `djinterop::unsupported_operation`. +/// +/// This header is the high-level API, which presents a device through the +/// format-agnostic `database` interface. The low-level API, which exposes +/// the tables of the format as they are, is in `onelibrary/v1`. namespace djinterop::onelibrary { /// Location of the library database within a device, relative to its root. @@ -65,8 +67,7 @@ bool DJINTEROP_PUBLIC database_exists(const std::string& path); /// \return Returns the loaded database. /// \throws database_not_found If no database is present at the given path. /// \throws unsupported_database If the passphrase does not open the database, -/// or the SQLite in use is older than 3.36, which -/// is the oldest that can read one. +/// or this build of libdjinterop cannot read one. /// \throws database_inconsistency If the database does not hold the tables /// that a OneLibrary database is expected to. database DJINTEROP_PUBLIC load_database( @@ -76,76 +77,6 @@ database DJINTEROP_PUBLIC load_database( /// State shared by everything belonging to one loaded database. struct onelibrary_context; -/// A loaded OneLibrary device, with access to the parts of the format that -/// the format-agnostic `database` interface has nowhere to put. -/// -/// A device is loaded once and read many times: decryption derives a key, -/// which is deliberately expensive, and then holds the whole database in -/// memory, so a caller that wants both the unified interface and the extras -/// below should load a `library` and take `db()` from it rather than also -/// calling `load_database`. -class DJINTEROP_PUBLIC library -{ -public: - /// Load the database on a device. - /// - /// Arguments and exceptions are those of `load_database`. - explicit library( - const std::string& path, - const std::string& passphrase = default_passphrase); - - /// The device, through the format-agnostic interface. - [[nodiscard]] database db() const; - - /// Root directory of the device, to which every path is relative. - /// - /// This is the directory that was loaded, and not the directory the - /// database file itself sits in. - [[nodiscard]] const std::string& directory() const; - - /// Path of the ANLZ analysis data for a track, relative to `directory()`. - /// - /// rekordbox keeps the beatgrid, cues, loops and waveforms out of the - /// database and in a set of files beside the music, which this names: the - /// `.DAT` file given here, and the siblings that differ from it only in - /// extension. `.EXT` holds the colour waveform and the cues beyond the - /// first three, and `.2EX` the waveform that a touch display draws. - /// - /// \param track_id Identifier of the track, as `track::id()` gives it. - /// \return Returns the path, or no value if the track is not there or - /// carries no analysis data. - [[nodiscard]] std::optional analysis_path( - int64_t track_id) const; - - /// The musical key of a track, in the notation the device records. - /// - /// `track::key()` gives the key as one of the twenty-four this library - /// knows, which loses both the notation rekordbox wrote and any key whose - /// notation is not recognised. This gives back what the device holds, - /// such as `F#m` under the classic setting or `8A` under the Camelot one, - /// for a caller that would rather read it itself. - /// - /// \param track_id Identifier of the track, as `track::id()` gives it. - /// \return Returns the notation, or no value if the track is not there or - /// carries no key. - [[nodiscard]] std::optional key_name(int64_t track_id) const; - - /// The colour a track is marked with, as rekordbox enumerates them. - /// - /// The colours are the eight that rekordbox offers, numbered as the - /// `export.pdb` library beside the database numbers them: one for pink, - /// two red, three orange, four yellow, five green, six aqua, seven blue - /// and eight purple. - /// - /// \param track_id Identifier of the track, as `track::id()` gives it. - /// \return Returns the colour, or no value if the track is not there or - /// is not marked with one. - [[nodiscard]] std::optional color_id(int64_t track_id) const; - -private: - std::shared_ptr context_; -}; - } // namespace djinterop::onelibrary #endif // DJINTEROP_ONELIBRARY_ONELIBRARY_HPP diff --git a/include/djinterop/onelibrary/v1/content_table.hpp b/include/djinterop/onelibrary/v1/content_table.hpp new file mode 100644 index 0000000..1a9f1a2 --- /dev/null +++ b/include/djinterop/onelibrary/v1/content_table.hpp @@ -0,0 +1,182 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once +#ifndef DJINTEROP_ONELIBRARY_V1_CONTENT_TABLE_HPP +#define DJINTEROP_ONELIBRARY_V1_CONTENT_TABLE_HPP + +#include +#include +#include +#include +#include +#include + +#include + +namespace djinterop::onelibrary +{ +struct onelibrary_context; + +namespace v1 +{ +/// Special value for id to indicate that a given row is not a row of the +/// database. Every row read from a device carries a real identifier, so this +/// is only ever the value a default-constructed row holds. +constexpr int64_t CONTENT_ROW_ID_NONE = 0; + +/// Special value for the `color_id` column that indicates that a track is not +/// marked with a colour. +constexpr int64_t COLOR_ID_NONE = 0; + +/// One row of the `content` table, with its lookup tables resolved. +/// +/// The schema declares no constraints: nothing is `NOT NULL` and no foreign +/// key is enforced, so every field is optional and a lookup reference may +/// point at a row that is not there. +struct content_row +{ + int64_t id = CONTENT_ROW_ID_NONE; + + std::optional title; + std::optional artist; + std::optional composer; + std::optional album; + std::optional genre; + + /// The record label, which djinterop calls the publisher. + std::optional label; + + /// The musical key in the notation rekordbox wrote, which follows the + /// setting it exported under and may be Camelot, such as `8A`. + std::optional key; + + std::optional comment; + + /// Tempo in hundredths of a beat per minute: 12400 is 124.00 BPM. + std::optional bpm_x100; + + /// Duration, which the database records in whole seconds. + /// + /// The published description of the format calls this column + /// milliseconds, which it is not: on a real export, a 320 kbps track of + /// 6,517,615 bytes carries a `length` of 161, and 6517615 * 8 / 320000 is + /// 163 seconds. No other reading fits. + std::optional length; + + std::optional track_number; + std::optional release_year; + + /// Rating in whole stars, from zero to five. + /// + /// This is not the 0-255 encoding the rest of the rekordbox ecosystem + /// uses, whose only values are 0, 51, 102, 153, 204 and 255. A real + /// export holds a 5, which that encoding cannot express. + std::optional rating_stars; + + /// POSIX path, absolute within the device and so beginning with a + /// separator, such as `/Contents/Artist/Album/Track.mp3`. + std::optional path; + + /// Size of the file, in bytes. + std::optional file_size; + + /// Bitrate of the file, in kilobits per second. + std::optional bitrate; + + /// Sampling rate of the file, in hertz. + std::optional sampling_rate; + + /// Path of the ANLZ analysis data, as the device records it. + /// + /// rekordbox keeps the beatgrid, cues, loops and waveforms in files beside + /// the music rather than in the database. This names the `.DAT` one; its + /// siblings differ only in extension, `.EXT` holding the colour waveform + /// and the cues beyond the first three, and `.2EX` the waveform a touch + /// display draws. + std::optional analysis_path; + + /// The colour the track is marked with, or `COLOR_ID_NONE` for none. + /// + /// The eight colours rekordbox offers, numbered as the `export.pdb` + /// library beside the database numbers them: one for pink, two red, three + /// orange, four yellow, five green, six aqua, seven blue, eight purple. + std::optional color_id; +}; + +/// Read access to the `content` table and the lookup tables it references. +/// +/// A whole row can be read at once, or any one column on its own, which +/// avoids the joins that resolving every lookup table costs. +class DJINTEROP_PUBLIC content_table +{ +public: + explicit content_table(std::shared_ptr context); + + /// Fetch one row by its identifier. + [[nodiscard]] std::optional get(int64_t id) const; + + /// Fetch the identifiers of every row, ordered. + [[nodiscard]] std::vector all_ids() const; + + /// Fetch the identifiers of rows whose path matches, ordered. + /// + /// Paths in the database are absolute within the device and begin with a + /// separator; a path given without one is matched as though it had one. + [[nodiscard]] std::vector ids_by_path( + const std::string& path) const; + + /// Test whether a row exists. + [[nodiscard]] bool exists(int64_t id) const; + + /// Fetch one column of one row. + /// + /// Each returns no value if the row is not there or the column is unset. + /// Text that is set but empty counts as unset, which is how rekordbox + /// writes metadata a track does not carry. + /// @{ + [[nodiscard]] std::optional get_title(int64_t id) const; + [[nodiscard]] std::optional get_artist(int64_t id) const; + [[nodiscard]] std::optional get_composer(int64_t id) const; + [[nodiscard]] std::optional get_album(int64_t id) const; + [[nodiscard]] std::optional get_genre(int64_t id) const; + [[nodiscard]] std::optional get_label(int64_t id) const; + [[nodiscard]] std::optional get_key(int64_t id) const; + [[nodiscard]] std::optional get_comment(int64_t id) const; + [[nodiscard]] std::optional get_bpm_x100(int64_t id) const; + [[nodiscard]] std::optional get_length( + int64_t id) const; + [[nodiscard]] std::optional get_track_number(int64_t id) const; + [[nodiscard]] std::optional get_release_year(int64_t id) const; + [[nodiscard]] std::optional get_rating_stars(int64_t id) const; + [[nodiscard]] std::optional get_path(int64_t id) const; + [[nodiscard]] std::optional get_file_size(int64_t id) const; + [[nodiscard]] std::optional get_bitrate(int64_t id) const; + [[nodiscard]] std::optional get_sampling_rate(int64_t id) const; + [[nodiscard]] std::optional get_analysis_path( + int64_t id) const; + [[nodiscard]] std::optional get_color_id(int64_t id) const; + /// @} + +private: + std::shared_ptr context_; +}; + +} // namespace v1 +} // namespace djinterop::onelibrary + +#endif // DJINTEROP_ONELIBRARY_V1_CONTENT_TABLE_HPP diff --git a/include/djinterop/onelibrary/v1/library.hpp b/include/djinterop/onelibrary/v1/library.hpp new file mode 100644 index 0000000..ceff539 --- /dev/null +++ b/include/djinterop/onelibrary/v1/library.hpp @@ -0,0 +1,78 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once +#ifndef DJINTEROP_ONELIBRARY_V1_LIBRARY_HPP +#define DJINTEROP_ONELIBRARY_V1_LIBRARY_HPP + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace djinterop::onelibrary::v1 +{ +/// A loaded OneLibrary device, through the low-level API. +/// +/// The tables expose the format as the device holds it, translating no +/// further than resolving a lookup reference to the text behind it. +/// +/// Loading decrypts the device into memory, and derives a key to do it, which +/// is deliberately expensive; take `database()` from a library rather than +/// also calling `load_database`. +class DJINTEROP_PUBLIC library +{ +public: + /// Load the database on a device, as `load_database` does. + explicit library( + const std::string& path, + const std::string& passphrase = default_passphrase); + + /// Construct from a context, as `load_database` builds one. + explicit library(std::shared_ptr context); + + /// The `content` table. + [[nodiscard]] content_table content() const { return content_; } + + /// The `playlist` table, and the membership beside it. + [[nodiscard]] playlist_table playlist() const { return playlist_; } + + /// The `property` table, which records the schema version. + [[nodiscard]] property_table property() const { return property_; } + + /// The device, through the format-agnostic interface. + [[nodiscard]] djinterop::database database() const; + + /// Root directory of the device, to which every path is relative, and + /// not the directory the database file itself sits in. + [[nodiscard]] const std::string& directory() const; + +private: + std::shared_ptr context_; + content_table content_; + playlist_table playlist_; + property_table property_; +}; + +} // namespace djinterop::onelibrary::v1 + +#endif // DJINTEROP_ONELIBRARY_V1_LIBRARY_HPP diff --git a/src/djinterop/onelibrary/playlist_table.hpp b/include/djinterop/onelibrary/v1/playlist_table.hpp similarity index 76% rename from src/djinterop/onelibrary/playlist_table.hpp rename to include/djinterop/onelibrary/v1/playlist_table.hpp index f4e32eb..b0e64e1 100644 --- a/src/djinterop/onelibrary/playlist_table.hpp +++ b/include/djinterop/onelibrary/v1/playlist_table.hpp @@ -16,6 +16,8 @@ */ #pragma once +#ifndef DJINTEROP_ONELIBRARY_V1_PLAYLIST_TABLE_HPP +#define DJINTEROP_ONELIBRARY_V1_PLAYLIST_TABLE_HPP #include #include @@ -23,31 +25,34 @@ #include #include -#include "onelibrary_context.hpp" +#include namespace djinterop::onelibrary { +struct onelibrary_context; + +namespace v1 +{ +/// Special value for id to indicate that a given row is not a row of the +/// database, and the value a root playlist carries as its parent. +constexpr int64_t PLAYLIST_ROW_ID_NONE = 0; + /// One row of the `playlist` table. struct playlist_row { - int64_t id = 0; + int64_t id = PLAYLIST_ROW_ID_NONE; std::string name; - /// The playlist this one sits under, if any. - /// - /// A root playlist has either no parent recorded or a parent of zero: the - /// schema enforces no foreign key, and rekordbox writes both. + /// The playlist this one sits under. A root has either no parent + /// recorded or a parent of `PLAYLIST_ROW_ID_NONE`; rekordbox writes both. std::optional parent_id; /// Position among siblings, counting from one. std::optional sequence_number; }; -/// Read access to the playlist tree and its membership. -/// -/// OneLibrary has a single tree of playlists, which libdjinterop presents both -/// as playlists and as crates; the two are not distinct in this format. -class playlist_table +/// Read access to the `playlist` tree and the `playlist_content` membership. +class DJINTEROP_PUBLIC playlist_table { public: explicit playlist_table(std::shared_ptr context); @@ -84,4 +89,7 @@ class playlist_table std::shared_ptr context_; }; +} // namespace v1 } // namespace djinterop::onelibrary + +#endif // DJINTEROP_ONELIBRARY_V1_PLAYLIST_TABLE_HPP diff --git a/include/djinterop/onelibrary/v1/property_table.hpp b/include/djinterop/onelibrary/v1/property_table.hpp new file mode 100644 index 0000000..1c922dd --- /dev/null +++ b/include/djinterop/onelibrary/v1/property_table.hpp @@ -0,0 +1,71 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once +#ifndef DJINTEROP_ONELIBRARY_V1_PROPERTY_TABLE_HPP +#define DJINTEROP_ONELIBRARY_V1_PROPERTY_TABLE_HPP + +#include +#include +#include +#include + +#include + +namespace djinterop::onelibrary +{ +struct onelibrary_context; + +namespace v1 +{ +/// The `dbVersion` that this implementation was written against. +constexpr const char* supported_db_version = "1000"; + +/// The single row of the `property` table, which describes the device. +struct property_row +{ + std::optional device_name; + + /// Version of the schema, such as `1000`. + std::optional db_version; + + /// Number of tracks the device was exported with, which need not be the + /// number the `content` table now holds. + std::optional number_of_contents; + + std::optional created_date; +}; + +/// Read access to the `property` table. +class DJINTEROP_PUBLIC property_table +{ +public: + explicit property_table(std::shared_ptr context); + + /// The row, or no value if the table is empty. + [[nodiscard]] std::optional get() const; + + [[nodiscard]] std::optional get_db_version() const; + +private: + std::shared_ptr context_; +}; + +} // namespace v1 +} // namespace djinterop::onelibrary + +#endif // DJINTEROP_ONELIBRARY_V1_PROPERTY_TABLE_HPP diff --git a/src/djinterop/onelibrary/content_table.hpp b/src/djinterop/onelibrary/content_table.hpp deleted file mode 100644 index 00e5341..0000000 --- a/src/djinterop/onelibrary/content_table.hpp +++ /dev/null @@ -1,124 +0,0 @@ -/* - This file is part of libdjinterop. - - libdjinterop is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libdjinterop is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with libdjinterop. If not, see . - */ - -#pragma once - -#include -#include -#include -#include -#include - -#include - -#include "onelibrary_context.hpp" - -namespace djinterop::onelibrary -{ -/// One row of the `content` table, with its lookup tables resolved. -/// -/// The schema declares no constraints at all: nothing is `NOT NULL` and no -/// foreign key is enforced, so every field is optional and a lookup reference -/// may point at a row that is not there. -struct content_row -{ - int64_t id = 0; - - std::optional title; - std::optional artist; - std::optional composer; - std::optional album; - std::optional genre; - - /// The record label, which djinterop calls the publisher. - std::optional label; - - /// The musical key, in the notation rekordbox writes, such as `F#m`. - /// - /// Which notation a device carries follows the setting rekordbox exported - /// it under, so it may equally be Camelot, such as `8A`. - std::optional key; - - /// Free-text comment the DJ attached to the track. - std::optional comment; - - /// Tempo in hundredths of a beat per minute: 12400 is 124.00 BPM. - std::optional bpm_x100; - - /// Duration in whole seconds. - /// - /// The published description of the format calls this column - /// milliseconds, which it is not: on an export written by rekordbox, a - /// 320 kbps track of 6,517,615 bytes carries a `length` of 161, and - /// 6517615 * 8 / 320000 is 163 seconds. Every track on that device - /// agrees to within a second, and none agrees on any other reading. - std::optional length_seconds; - - std::optional track_number; - std::optional release_year; - - /// Rating from zero to five stars, and not the 0-255 encoding that the - /// rest of the rekordbox ecosystem uses: an export written by rekordbox - /// holds only 0 and 5, and 5 is not a value that encoding can take. - std::optional rating_stars; - - /// Device-relative POSIX path, such as - /// `/Contents/Artist/Album/Track.mp3`. - std::optional path; - - std::optional file_size; - std::optional bitrate; - std::optional sampling_rate; -}; - -/// Read access to the `content` table and the lookup tables it references. -class content_table -{ -public: - explicit content_table(std::shared_ptr context); - - /// Fetch one row by its identifier. - [[nodiscard]] std::optional get(int64_t id) const; - - /// Fetch the identifiers of every row, ordered. - [[nodiscard]] std::vector all_ids() const; - - /// Fetch the identifiers of rows whose path matches, ordered. - /// - /// Paths in the database are absolute within the device and begin with a - /// separator; a path given without one is matched as though it had one. - [[nodiscard]] std::vector ids_by_path( - const std::string& path) const; - - /// Test whether a row exists. - [[nodiscard]] bool exists(int64_t id) const; - -private: - std::shared_ptr context_; -}; - -/// Build a track snapshot from a content row. -[[nodiscard]] track_snapshot to_snapshot(const content_row& row); - -/// Interpret the key notation that rekordbox writes, such as `F#m` or `Bb`. -/// -/// Returns no value for a notation that is not recognised, rather than -/// guessing. -[[nodiscard]] std::optional parse_musical_key( - const std::string& name); - -} // namespace djinterop::onelibrary diff --git a/src/djinterop/onelibrary/loader.hpp b/src/djinterop/onelibrary/loader.hpp new file mode 100644 index 0000000..cb4520b --- /dev/null +++ b/src/djinterop/onelibrary/loader.hpp @@ -0,0 +1,33 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include +#include + +namespace djinterop::onelibrary +{ +struct onelibrary_context; + +/// Decrypt the database on a device and check that it is one. +/// +/// \param path Either the root directory of a device, or the database file. +std::shared_ptr load_context( + const std::string& path, const std::string& passphrase); + +} // namespace djinterop::onelibrary diff --git a/src/djinterop/onelibrary/onelibrary.cpp b/src/djinterop/onelibrary/onelibrary.cpp index 2518a18..7b7ca99 100644 --- a/src/djinterop/onelibrary/onelibrary.cpp +++ b/src/djinterop/onelibrary/onelibrary.cpp @@ -24,8 +24,9 @@ #include "../util/crypto/encrypted_database.hpp" #include "../util/filesystem.hpp" -#include "database_impl.hpp" +#include "loader.hpp" #include "onelibrary_context.hpp" +#include "v1/database_impl.hpp" namespace djinterop::onelibrary { @@ -36,11 +37,10 @@ struct resolved_location /// Root directory of the device, to which track paths are relative. std::string directory; - /// The database file itself. std::string database_path; }; -/// Work out where the database is, given either a device or a database file. +/// Work out where the database is, given a device or the file itself. resolved_location resolve(const std::string& path) { // A path that names the database directly implies its device root, which @@ -53,9 +53,8 @@ resolved_location resolve(const std::string& path) auto end = path.size(); for (int level = 0; level < 3; ++level) { - const auto separator = - end == 0 ? std::string::npos - : path.find_last_of("/\\", end - 1); + const auto separator = end == 0 ? std::string::npos + : path.find_last_of("/\\", end - 1); // A relative path with nothing above it sits in the working // directory, which is then the root of the device. @@ -71,23 +70,8 @@ resolved_location resolve(const std::string& path) return resolved_location{path, path + "/" + database_relative_path}; } -/// Read one column of a track's row, if the row is there and the column set. -/// -/// The columns behind `library` are each read on their own, rather than -/// through `content_table`, whose query joins six lookup tables to build a -/// whole row that none of them needs. -template -std::optional track_column( - onelibrary_context& context, const char* sql, int64_t track_id) -{ - std::optional result; - context.db << sql << track_id >> - [&](std::optional value) { result = std::move(value); }; - - return result; -} +} // anonymous namespace -/// Decrypt the database on a device and check that it is one. std::shared_ptr load_context( const std::string& path, const std::string& passphrase) { @@ -96,16 +80,14 @@ std::shared_ptr load_context( if (!util::path_exists(location.database_path)) throw database_not_found{location.database_path}; - // Key derivation is deliberately expensive, so the passphrase is not - // tested separately: a wrong one shows up as the database failing to open, - // and is reported as such. + // Key derivation is expensive, so the passphrase is not tested + // separately: a wrong one shows up as the database failing to open. std::shared_ptr context; try { context = std::make_shared( - location.directory, - util::crypto::open_encrypted_database( - location.database_path, passphrase)); + location.directory, util::crypto::open_encrypted_database( + location.database_path, passphrase)); // Opening a database reads nothing, so touch it here: a wrong // passphrase would otherwise not be noticed until the first query. @@ -126,15 +108,12 @@ std::shared_ptr load_context( throw unsupported_database{e.what()}; } - // Fail here, while the caller still has the path in hand, rather than at - // the first query. - database_impl{context}.verify(); + // Fail here, while the caller still has the path in hand. + v1::database_impl{context}.verify(); return context; } -} // anonymous namespace - bool database_exists(const std::string& path) { const auto location = resolve(path); @@ -143,62 +122,8 @@ bool database_exists(const std::string& path) database load_database(const std::string& path, const std::string& passphrase) { - return database{std::make_shared( - load_context(path, passphrase))}; -} - -library::library(const std::string& path, const std::string& passphrase) : - context_{load_context(path, passphrase)} -{ -} - -database library::db() const -{ - return database{std::make_shared(context_)}; -} - -const std::string& library::directory() const -{ - return context_->directory; -} - -std::optional library::analysis_path(int64_t track_id) const -{ - const auto path = track_column( - *context_, "SELECT analysisDataFilePath FROM content WHERE content_id = ?", - track_id); - if (!path || path->empty()) - return std::nullopt; - - // Paths are absolute within the device, whereas every path this library - // hands out is relative to its root. - return path->front() == '/' ? path->substr(1) : *path; -} - -std::optional library::key_name(int64_t track_id) const -{ - const auto name = track_column( - *context_, - "SELECT \"key\".name FROM content AS c " - // `key` is quoted throughout, as it is also a SQL keyword. - "LEFT JOIN \"key\" ON \"key\".key_id = c.key_id " - "WHERE c.content_id = ?", - track_id); - if (!name || name->empty()) - return std::nullopt; - - return name; -} - -std::optional library::color_id(int64_t track_id) const -{ - const auto id = track_column( - *context_, "SELECT color_id FROM content WHERE content_id = ?", - track_id); - if (!id || *id == 0) - return std::nullopt; - - return static_cast(*id); + return database{ + std::make_shared(load_context(path, passphrase))}; } } // namespace djinterop::onelibrary diff --git a/src/djinterop/onelibrary/onelibrary_context.hpp b/src/djinterop/onelibrary/onelibrary_context.hpp index 1379967..944d634 100644 --- a/src/djinterop/onelibrary/onelibrary_context.hpp +++ b/src/djinterop/onelibrary/onelibrary_context.hpp @@ -17,11 +17,8 @@ #pragma once -#include -#include #include #include -#include #include @@ -54,35 +51,4 @@ struct onelibrary_context "OneLibrary databases are currently read-only in libdjinterop"}; } -/// Run a query whose rows are each a single identifier. -template -std::vector collect_ids( - onelibrary_context& context, const char* sql, const Args&... args) -{ - std::vector results; - auto query = context.db << sql; - ((query << args), ...); - query >> [&](int64_t id) { results.push_back(id); }; - return results; -} - -/// Run a query whose rows are each a single identifier, and take the first. -template -std::optional first_id( - onelibrary_context& context, const char* sql, const Args&... args) -{ - std::optional result; - auto query = context.db << sql; - ((query << args), ...); - query >> [&](int64_t id) { result = id; }; - return result; -} - -/// Test whether a query matches any row at all. -template -bool any_row(onelibrary_context& context, const char* sql, const Args&... args) -{ - return first_id(context, sql, args...).has_value(); -} - } // namespace djinterop::onelibrary diff --git a/src/djinterop/onelibrary/v1/content_table.cpp b/src/djinterop/onelibrary/v1/content_table.cpp new file mode 100644 index 0000000..240bcbc --- /dev/null +++ b/src/djinterop/onelibrary/v1/content_table.cpp @@ -0,0 +1,281 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#include + +#include +#include + +#include "../../util/sqlite_query.hpp" +#include "../onelibrary_context.hpp" + +namespace djinterop::onelibrary::v1 +{ +namespace +{ +/// How each lookup table is joined to `content`. +/// +/// Nothing enforces that a reference resolves, so each is joined outwards. +/// `key` is quoted throughout, as it is also a SQL keyword. +constexpr const char* artist_join = + "artist ON artist.artist_id = c.artist_id_artist"; +constexpr const char* composer_join = + "artist AS composer ON composer.artist_id = c.artist_id_composer"; +constexpr const char* album_join = "album ON album.album_id = c.album_id"; +constexpr const char* genre_join = "genre ON genre.genre_id = c.genre_id"; +constexpr const char* label_join = "label ON label.label_id = c.label_id"; +constexpr const char* key_join = "\"key\" ON \"key\".key_id = c.key_id"; + +/// Every column the row structure needs, in the order it reads them back. +constexpr const char* row_columns = + "c.content_id, c.title, artist.name, composer.name, album.name, " + "genre.name, label.name, \"key\".name, c.djComment, c.bpmx100, c.length, " + "c.trackNo, c.releaseYear, c.rating, c.path, c.fileSize, " + "c.bitrate, c.samplingRate, c.analysisDataFilePath, c.color_id"; + +std::string row_query() +{ + return std::string{"SELECT "} + row_columns + " FROM content AS c " + + "LEFT JOIN " + artist_join + " LEFT JOIN " + composer_join + + " LEFT JOIN " + album_join + " LEFT JOIN " + genre_join + + " LEFT JOIN " + label_join + " LEFT JOIN " + key_join + + " WHERE c.content_id = ?"; +} + +/// A query for one column of `content`. +std::string content_column(const std::string& expression) +{ + return "SELECT " + expression + " FROM content AS c WHERE c.content_id = ?"; +} + +/// A query for the name a lookup table holds. +std::string lookup_column(const std::string& alias, const std::string& join) +{ + return "SELECT " + alias + ".name FROM content AS c LEFT JOIN " + join + + " WHERE c.content_id = ?"; +} + +/// Read one column, if the row is there. +template +std::optional read_column( + onelibrary_context& context, const std::string& sql, int64_t id) +{ + std::optional result; + context.db << sql << id >> [&](std::optional value) + { result = std::move(value); }; + + return result; +} + +/// Treat a column that is present but empty as absent: rekordbox writes an +/// empty string for metadata a track does not carry. +std::optional non_empty(std::optional value) +{ + if (value.has_value() && value->empty()) + return std::nullopt; + + return value; +} + +/// Read one column of text, which is absent when it is empty. +std::optional read_text( + onelibrary_context& context, const std::string& sql, int64_t id) +{ + return non_empty(read_column(context, sql, id)); +} + +} // anonymous namespace + +content_table::content_table(std::shared_ptr context) : + context_{std::move(context)} +{ +} + +std::optional content_table::get(int64_t id) const +{ + std::optional result; + + // The parameter list has to match `row_columns` exactly. + context_->db << row_query() << id >> + [&](int64_t row_id, std::optional title, + std::optional artist, + std::optional composer, + std::optional album, std::optional genre, + std::optional label, std::optional key, + std::optional comment, std::optional bpm_x100, + std::optional length, std::optional track_number, + std::optional release_year, + std::optional rating_stars, + std::optional path, std::optional file_size, + std::optional bitrate, + std::optional sampling_rate, + std::optional analysis_path, + std::optional color_id) + { + content_row row; + row.id = row_id; + row.title = non_empty(std::move(title)); + row.artist = non_empty(std::move(artist)); + row.composer = non_empty(std::move(composer)); + row.album = non_empty(std::move(album)); + row.genre = non_empty(std::move(genre)); + row.label = non_empty(std::move(label)); + row.key = non_empty(std::move(key)); + row.comment = non_empty(std::move(comment)); + row.bpm_x100 = bpm_x100; + if (length) + row.length = std::chrono::seconds{*length}; + row.track_number = track_number; + row.release_year = release_year; + row.rating_stars = rating_stars; + row.path = non_empty(std::move(path)); + row.file_size = file_size; + row.bitrate = bitrate; + row.sampling_rate = sampling_rate; + row.analysis_path = non_empty(std::move(analysis_path)); + row.color_id = color_id; + result = std::move(row); + }; + + return result; +} + +std::vector content_table::all_ids() const +{ + return util::collect_ids( + context_->db, "SELECT content_id FROM content ORDER BY content_id"); +} + +std::vector content_table::ids_by_path(const std::string& path) const +{ + const auto qualified = + !path.empty() && path.front() == '/' ? path : "/" + path; + + return util::collect_ids( + context_->db, + "SELECT content_id FROM content WHERE path = ? ORDER BY content_id", + qualified); +} + +bool content_table::exists(int64_t id) const +{ + return util::any_row( + context_->db, "SELECT 1 FROM content WHERE content_id = ? LIMIT 1", id); +} + +std::optional content_table::get_title(int64_t id) const +{ + return read_text(*context_, content_column("c.title"), id); +} + +std::optional content_table::get_artist(int64_t id) const +{ + return read_text(*context_, lookup_column("artist", artist_join), id); +} + +std::optional content_table::get_composer(int64_t id) const +{ + return read_text(*context_, lookup_column("composer", composer_join), id); +} + +std::optional content_table::get_album(int64_t id) const +{ + return read_text(*context_, lookup_column("album", album_join), id); +} + +std::optional content_table::get_genre(int64_t id) const +{ + return read_text(*context_, lookup_column("genre", genre_join), id); +} + +std::optional content_table::get_label(int64_t id) const +{ + return read_text(*context_, lookup_column("label", label_join), id); +} + +std::optional content_table::get_key(int64_t id) const +{ + return read_text(*context_, lookup_column("\"key\"", key_join), id); +} + +std::optional content_table::get_comment(int64_t id) const +{ + return read_text(*context_, content_column("c.djComment"), id); +} + +std::optional content_table::get_bpm_x100(int64_t id) const +{ + return read_column(*context_, content_column("c.bpmx100"), id); +} + +std::optional content_table::get_length(int64_t id) const +{ + const auto seconds = + read_column(*context_, content_column("c.length"), id); + if (!seconds) + return std::nullopt; + + return std::chrono::seconds{*seconds}; +} + +std::optional content_table::get_track_number(int64_t id) const +{ + return read_column(*context_, content_column("c.trackNo"), id); +} + +std::optional content_table::get_release_year(int64_t id) const +{ + return read_column(*context_, content_column("c.releaseYear"), id); +} + +std::optional content_table::get_rating_stars(int64_t id) const +{ + return read_column(*context_, content_column("c.rating"), id); +} + +std::optional content_table::get_path(int64_t id) const +{ + return read_text(*context_, content_column("c.path"), id); +} + +std::optional content_table::get_file_size(int64_t id) const +{ + return read_column(*context_, content_column("c.fileSize"), id); +} + +std::optional content_table::get_bitrate(int64_t id) const +{ + return read_column(*context_, content_column("c.bitrate"), id); +} + +std::optional content_table::get_sampling_rate(int64_t id) const +{ + return read_column( + *context_, content_column("c.samplingRate"), id); +} + +std::optional content_table::get_analysis_path(int64_t id) const +{ + return read_text(*context_, content_column("c.analysisDataFilePath"), id); +} + +std::optional content_table::get_color_id(int64_t id) const +{ + return read_column(*context_, content_column("c.color_id"), id); +} + +} // namespace djinterop::onelibrary::v1 diff --git a/src/djinterop/onelibrary/crate_impl.cpp b/src/djinterop/onelibrary/v1/crate_impl.cpp similarity index 96% rename from src/djinterop/onelibrary/crate_impl.cpp rename to src/djinterop/onelibrary/v1/crate_impl.cpp index 3c798b8..e8f5b55 100644 --- a/src/djinterop/onelibrary/crate_impl.cpp +++ b/src/djinterop/onelibrary/v1/crate_impl.cpp @@ -21,13 +21,12 @@ #include #include +#include #include #include "database_impl.hpp" -#include "playlist_table.hpp" #include "track_impl.hpp" - -namespace djinterop::onelibrary +namespace djinterop::onelibrary::v1 { crate_impl::crate_impl( std::shared_ptr context, int64_t id) : @@ -148,4 +147,4 @@ void crate_impl::set_parent(std::optional) read_only(); } -} // namespace djinterop::onelibrary +} // namespace djinterop::onelibrary::v1 diff --git a/src/djinterop/onelibrary/crate_impl.hpp b/src/djinterop/onelibrary/v1/crate_impl.hpp similarity index 93% rename from src/djinterop/onelibrary/crate_impl.hpp rename to src/djinterop/onelibrary/v1/crate_impl.hpp index da8c532..168ead8 100644 --- a/src/djinterop/onelibrary/crate_impl.hpp +++ b/src/djinterop/onelibrary/v1/crate_impl.hpp @@ -18,15 +18,15 @@ #pragma once #include + #include #include #include #include -#include "../impl/crate_impl.hpp" -#include "onelibrary_context.hpp" - -namespace djinterop::onelibrary +#include "../../impl/crate_impl.hpp" +#include "../onelibrary_context.hpp" +namespace djinterop::onelibrary::v1 { /// A crate in a OneLibrary database. /// @@ -63,4 +63,4 @@ class crate_impl : public djinterop::crate_impl /// Wrap a playlist row as a crate. crate make_crate(std::shared_ptr context, int64_t id); -} // namespace djinterop::onelibrary +} // namespace djinterop::onelibrary::v1 diff --git a/src/djinterop/onelibrary/database_impl.cpp b/src/djinterop/onelibrary/v1/database_impl.cpp similarity index 94% rename from src/djinterop/onelibrary/database_impl.cpp rename to src/djinterop/onelibrary/v1/database_impl.cpp index 7569613..4d27b2d 100644 --- a/src/djinterop/onelibrary/database_impl.cpp +++ b/src/djinterop/onelibrary/v1/database_impl.cpp @@ -22,15 +22,15 @@ #include #include +#include +#include +#include #include -#include "content_table.hpp" #include "crate_impl.hpp" #include "playlist_impl.hpp" -#include "playlist_table.hpp" #include "track_impl.hpp" - -namespace djinterop::onelibrary +namespace djinterop::onelibrary::v1 { namespace { @@ -76,10 +76,8 @@ std::string database_impl::uuid() std::string database_impl::version_name() { - std::string version; - context_->db << "SELECT dbVersion FROM property LIMIT 1" >> - [&](std::optional db_version) - { version = db_version.value_or(std::string{}); }; + const auto version = + property_table{context_}.get_db_version().value_or(std::string{}); return version.empty() ? "OneLibrary" : "OneLibrary " + version; } @@ -211,4 +209,4 @@ void database_impl::remove_track(djinterop::track) read_only(); } -} // namespace djinterop::onelibrary +} // namespace djinterop::onelibrary::v1 diff --git a/src/djinterop/onelibrary/database_impl.hpp b/src/djinterop/onelibrary/v1/database_impl.hpp similarity index 94% rename from src/djinterop/onelibrary/database_impl.hpp rename to src/djinterop/onelibrary/v1/database_impl.hpp index 87840e6..1d5c8b9 100644 --- a/src/djinterop/onelibrary/database_impl.hpp +++ b/src/djinterop/onelibrary/v1/database_impl.hpp @@ -18,15 +18,15 @@ #pragma once #include + #include #include #include #include -#include "../impl/database_impl.hpp" -#include "onelibrary_context.hpp" - -namespace djinterop::onelibrary +#include "../../impl/database_impl.hpp" +#include "../onelibrary_context.hpp" +namespace djinterop::onelibrary::v1 { /// A loaded OneLibrary database, presented through the unified interface. class database_impl : public djinterop::database_impl @@ -65,4 +65,4 @@ class database_impl : public djinterop::database_impl std::shared_ptr context_; }; -} // namespace djinterop::onelibrary +} // namespace djinterop::onelibrary::v1 diff --git a/src/djinterop/onelibrary/v1/library.cpp b/src/djinterop/onelibrary/v1/library.cpp new file mode 100644 index 0000000..b53f7f7 --- /dev/null +++ b/src/djinterop/onelibrary/v1/library.cpp @@ -0,0 +1,48 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#include + +#include + +#include "../loader.hpp" +#include "../onelibrary_context.hpp" +#include "database_impl.hpp" +namespace djinterop::onelibrary::v1 +{ +library::library(const std::string& path, const std::string& passphrase) : + library{load_context(path, passphrase)} +{ +} + +library::library(std::shared_ptr context) : + context_{std::move(context)}, content_{context_}, playlist_{context_}, + property_{context_} +{ +} + +djinterop::database library::database() const +{ + return djinterop::database{std::make_shared(context_)}; +} + +const std::string& library::directory() const +{ + return context_->directory; +} + +} // namespace djinterop::onelibrary::v1 diff --git a/src/djinterop/onelibrary/playlist_impl.cpp b/src/djinterop/onelibrary/v1/playlist_impl.cpp similarity index 96% rename from src/djinterop/onelibrary/playlist_impl.cpp rename to src/djinterop/onelibrary/v1/playlist_impl.cpp index c7da06b..e17410e 100644 --- a/src/djinterop/onelibrary/playlist_impl.cpp +++ b/src/djinterop/onelibrary/v1/playlist_impl.cpp @@ -21,13 +21,12 @@ #include #include +#include #include #include "database_impl.hpp" -#include "playlist_table.hpp" #include "track_impl.hpp" - -namespace djinterop::onelibrary +namespace djinterop::onelibrary::v1 { playlist_impl::playlist_impl( std::shared_ptr context, int64_t id) : @@ -144,4 +143,4 @@ void playlist_impl::set_parent(const djinterop::playlist_impl*) read_only(); } -} // namespace djinterop::onelibrary +} // namespace djinterop::onelibrary::v1 diff --git a/src/djinterop/onelibrary/playlist_impl.hpp b/src/djinterop/onelibrary/v1/playlist_impl.hpp similarity index 94% rename from src/djinterop/onelibrary/playlist_impl.hpp rename to src/djinterop/onelibrary/v1/playlist_impl.hpp index b7d64ea..8eeccd1 100644 --- a/src/djinterop/onelibrary/playlist_impl.hpp +++ b/src/djinterop/onelibrary/v1/playlist_impl.hpp @@ -18,15 +18,15 @@ #pragma once #include + #include #include #include #include -#include "../impl/playlist_impl.hpp" -#include "onelibrary_context.hpp" - -namespace djinterop::onelibrary +#include "../../impl/playlist_impl.hpp" +#include "../onelibrary_context.hpp" +namespace djinterop::onelibrary::v1 { /// A playlist in a OneLibrary database. /// @@ -69,4 +69,4 @@ class playlist_impl : public djinterop::playlist_impl /// Wrap a playlist row as a playlist. playlist make_playlist(std::shared_ptr context, int64_t id); -} // namespace djinterop::onelibrary +} // namespace djinterop::onelibrary::v1 diff --git a/src/djinterop/onelibrary/playlist_table.cpp b/src/djinterop/onelibrary/v1/playlist_table.cpp similarity index 85% rename from src/djinterop/onelibrary/playlist_table.cpp rename to src/djinterop/onelibrary/v1/playlist_table.cpp index ae34c78..7a4fc39 100644 --- a/src/djinterop/onelibrary/playlist_table.cpp +++ b/src/djinterop/onelibrary/v1/playlist_table.cpp @@ -15,11 +15,13 @@ along with libdjinterop. If not, see . */ -#include "playlist_table.hpp" +#include #include -namespace djinterop::onelibrary +#include "../../util/sqlite_query.hpp" +#include "../onelibrary_context.hpp" +namespace djinterop::onelibrary::v1 { namespace { @@ -73,19 +75,20 @@ std::optional playlist_table::get(int64_t id) const bool playlist_table::exists(int64_t id) const { - return any_row( - *context_, "SELECT 1 FROM playlist WHERE playlist_id = ? LIMIT 1", id); + return util::any_row( + context_->db, "SELECT 1 FROM playlist WHERE playlist_id = ? LIMIT 1", + id); } std::vector playlist_table::root_ids() const { - return collect_ids(*context_, select_roots); + return util::collect_ids(context_->db, select_roots); } std::vector playlist_table::child_ids(int64_t id) const { - return collect_ids( - *context_, + return util::collect_ids( + context_->db, "SELECT playlist_id FROM playlist WHERE playlist_id_parent = ? " "ORDER BY sequenceNo, playlist_id", id); @@ -95,8 +98,8 @@ std::vector playlist_table::descendant_ids(int64_t id) const { // One recursive query rather than one per node. `depth` keeps the result // breadth first, and `sequenceNo` keeps siblings in their own order. - return collect_ids( - *context_, + return util::collect_ids( + context_->db, "WITH RECURSIVE descendant(playlist_id, sequenceNo, depth) AS (" "SELECT playlist_id, sequenceNo, 0 FROM playlist " "WHERE playlist_id_parent = ? " @@ -111,14 +114,14 @@ std::vector playlist_table::descendant_ids(int64_t id) const std::optional playlist_table::find_root(const std::string& name) const { - return first_id(*context_, find_root_by_name, name); + return util::first_id(context_->db, find_root_by_name, name); } std::optional playlist_table::find_child( int64_t parent_id, const std::string& name) const { - return first_id( - *context_, + return util::first_id( + context_->db, "SELECT playlist_id FROM playlist " "WHERE playlist_id_parent = ? AND name = ? " "ORDER BY sequenceNo, playlist_id LIMIT 1", @@ -127,8 +130,8 @@ std::optional playlist_table::find_child( std::vector playlist_table::track_ids(int64_t id) const { - return collect_ids( - *context_, + return util::collect_ids( + context_->db, "SELECT content_id FROM playlist_content WHERE playlist_id = ? " "ORDER BY sequenceNo, rowid", id); @@ -137,11 +140,11 @@ std::vector playlist_table::track_ids(int64_t id) const std::vector playlist_table::playlists_containing( int64_t track_id) const { - return collect_ids( - *context_, + return util::collect_ids( + context_->db, "SELECT DISTINCT playlist_id FROM playlist_content " "WHERE content_id = ? ORDER BY playlist_id", track_id); } -} // namespace djinterop::onelibrary +} // namespace djinterop::onelibrary::v1 diff --git a/src/djinterop/onelibrary/v1/property_table.cpp b/src/djinterop/onelibrary/v1/property_table.cpp new file mode 100644 index 0000000..a161798 --- /dev/null +++ b/src/djinterop/onelibrary/v1/property_table.cpp @@ -0,0 +1,62 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#include + +#include + +#include "../onelibrary_context.hpp" +namespace djinterop::onelibrary::v1 +{ +property_table::property_table(std::shared_ptr context) : + context_{std::move(context)} +{ +} + +std::optional property_table::get() const +{ + std::optional result; + + context_->db << "SELECT deviceName, dbVersion, numberOfContents, " + "createdDate FROM property LIMIT 1" >> + [&](std::optional device_name, + std::optional db_version, + std::optional number_of_contents, + std::optional created_date) + { + property_row row; + row.device_name = std::move(device_name); + row.db_version = std::move(db_version); + row.number_of_contents = number_of_contents; + row.created_date = std::move(created_date); + result = std::move(row); + }; + + return result; +} + +std::optional property_table::get_db_version() const +{ + std::optional result; + context_->db << "SELECT dbVersion FROM property LIMIT 1" >> + [&](std::optional version) + { result = std::move(version); }; + + return result; +} + +} // namespace djinterop::onelibrary::v1 diff --git a/src/djinterop/onelibrary/content_table.cpp b/src/djinterop/onelibrary/v1/track_conversion.cpp similarity index 56% rename from src/djinterop/onelibrary/content_table.cpp rename to src/djinterop/onelibrary/v1/track_conversion.cpp index 7e5fe02..ae2715d 100644 --- a/src/djinterop/onelibrary/content_table.cpp +++ b/src/djinterop/onelibrary/v1/track_conversion.cpp @@ -15,47 +15,15 @@ along with libdjinterop. If not, see . */ -#include "content_table.hpp" +#include "track_conversion.hpp" #include -#include #include -#include -#include - -namespace djinterop::onelibrary +#include +namespace djinterop::onelibrary::v1 { namespace { -/// Every column the row structure needs, in the order it reads them back. -/// -/// Nothing enforces that a lookup reference resolves, so each is joined -/// outwards. -constexpr const char* select_columns = - "SELECT c.content_id, c.title, artist.name, composer.name, album.name, " - "genre.name, label.name, \"key\".name, c.djComment, c.bpmx100, c.length, " - "c.trackNo, c.releaseYear, c.rating, c.path, c.fileSize, " - "c.bitrate, c.samplingRate " - "FROM content AS c " - "LEFT JOIN artist AS artist ON artist.artist_id = c.artist_id_artist " - "LEFT JOIN artist AS composer " - "ON composer.artist_id = c.artist_id_composer " - "LEFT JOIN album ON album.album_id = c.album_id " - "LEFT JOIN genre ON genre.genre_id = c.genre_id " - "LEFT JOIN label ON label.label_id = c.label_id " - // `key` is quoted throughout, as it is also a SQL keyword. - "LEFT JOIN \"key\" ON \"key\".key_id = c.key_id "; - -/// Treat a column that is present but empty as absent: rekordbox writes an -/// empty string for metadata a track does not carry. -std::optional non_empty(std::optional value) -{ - if (value.has_value() && value->empty()) - return std::nullopt; - - return value; -} - /// The offset in semitones of a note letter above C, if it is one. constexpr std::optional semitones_above_c(char note) { @@ -101,8 +69,9 @@ std::optional parse_musical_key(const std::string& name) if (position >= name.size()) return std::nullopt; - const auto note = semitones_above_c(static_cast( - std::toupper(static_cast(name[position])))); + const auto note = semitones_above_c( + static_cast( + std::toupper(static_cast(name[position])))); if (!note) return std::nullopt; @@ -141,80 +110,6 @@ std::optional parse_musical_key(const std::string& name) return std::nullopt; } -content_table::content_table(std::shared_ptr context) : - context_{std::move(context)} -{ -} - -std::optional content_table::get(int64_t id) const -{ - std::optional result; - - // The parameter list has to match `select_columns` exactly. - context_->db << (std::string{select_columns} + "WHERE c.content_id = ?") - << id >> - [&](int64_t row_id, std::optional title, - std::optional artist, - std::optional composer, - std::optional album, std::optional genre, - std::optional label, std::optional key, - std::optional comment, std::optional bpm_x100, - std::optional length_seconds, - std::optional track_number, - std::optional release_year, - std::optional rating_stars, - std::optional path, std::optional file_size, - std::optional bitrate, - std::optional sampling_rate) - { - content_row row; - row.id = row_id; - row.title = non_empty(std::move(title)); - row.artist = non_empty(std::move(artist)); - row.composer = non_empty(std::move(composer)); - row.album = non_empty(std::move(album)); - row.genre = non_empty(std::move(genre)); - row.label = non_empty(std::move(label)); - row.key = non_empty(std::move(key)); - row.comment = non_empty(std::move(comment)); - row.bpm_x100 = bpm_x100; - row.length_seconds = length_seconds; - row.track_number = track_number; - row.release_year = release_year; - row.rating_stars = rating_stars; - row.path = non_empty(std::move(path)); - row.file_size = file_size; - row.bitrate = bitrate; - row.sampling_rate = sampling_rate; - result = std::move(row); - }; - - return result; -} - -std::vector content_table::all_ids() const -{ - return collect_ids( - *context_, "SELECT content_id FROM content ORDER BY content_id"); -} - -std::vector content_table::ids_by_path(const std::string& path) const -{ - const auto qualified = - !path.empty() && path.front() == '/' ? path : "/" + path; - - return collect_ids( - *context_, - "SELECT content_id FROM content WHERE path = ? ORDER BY content_id", - qualified); -} - -bool content_table::exists(int64_t id) const -{ - return any_row( - *context_, "SELECT 1 FROM content WHERE content_id = ? LIMIT 1", id); -} - track_snapshot to_snapshot(const content_row& row) { track_snapshot snapshot; @@ -230,9 +125,9 @@ track_snapshot to_snapshot(const content_row& row) if (row.bpm_x100.has_value() && *row.bpm_x100 > 0) snapshot.bpm = static_cast(*row.bpm_x100) / 100; - if (row.length_seconds.has_value() && *row.length_seconds > 0) + if (row.length.has_value() && row.length->count() > 0) snapshot.duration = - std::chrono::milliseconds{*row.length_seconds * 1000}; + std::chrono::duration_cast(*row.length); if (row.track_number.has_value() && *row.track_number > 0) snapshot.track_number = static_cast(*row.track_number); @@ -266,9 +161,9 @@ track_snapshot to_snapshot(const content_row& row) // The database records a duration in whole seconds and no sample // count, so the count can only be recovered to that precision. - if (row.length_seconds.has_value() && *row.length_seconds > 0) + if (row.length.has_value() && row.length->count() > 0) snapshot.sample_count = static_cast( - *row.length_seconds * *row.sampling_rate); + row.length->count() * *row.sampling_rate); } if (row.key.has_value()) @@ -280,4 +175,4 @@ track_snapshot to_snapshot(const content_row& row) return snapshot; } -} // namespace djinterop::onelibrary +} // namespace djinterop::onelibrary::v1 diff --git a/src/djinterop/onelibrary/v1/track_conversion.hpp b/src/djinterop/onelibrary/v1/track_conversion.hpp new file mode 100644 index 0000000..198b52b --- /dev/null +++ b/src/djinterop/onelibrary/v1/track_conversion.hpp @@ -0,0 +1,39 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include + +#include + +#include +#include +#include +namespace djinterop::onelibrary::v1 +{ +/// Build a track snapshot from a content row. +[[nodiscard]] track_snapshot to_snapshot(const content_row& row); + +/// Interpret the key notation that rekordbox writes, such as `F#m` or `Bb`. +/// +/// Returns no value for a notation that is not recognised, rather than +/// guessing. +[[nodiscard]] std::optional parse_musical_key( + const std::string& name); + +} // namespace djinterop::onelibrary::v1 diff --git a/src/djinterop/onelibrary/track_impl.cpp b/src/djinterop/onelibrary/v1/track_impl.cpp similarity index 67% rename from src/djinterop/onelibrary/track_impl.cpp rename to src/djinterop/onelibrary/v1/track_impl.cpp index 3e5117f..6c80810 100644 --- a/src/djinterop/onelibrary/track_impl.cpp +++ b/src/djinterop/onelibrary/v1/track_impl.cpp @@ -17,18 +17,22 @@ #include "track_impl.hpp" +#include +#include +#include #include #include #include +#include +#include -#include "../util/filesystem.hpp" -#include "content_table.hpp" +#include "../../util/filesystem.hpp" #include "crate_impl.hpp" #include "database_impl.hpp" -#include "playlist_table.hpp" +#include "track_conversion.hpp" -namespace djinterop::onelibrary +namespace djinterop::onelibrary::v1 { // Loudness, beatgrids, waveforms, hot cues and loops are absent throughout: // rekordbox leaves them in the ANLZ files that `analysisDataFilePath` points @@ -74,8 +78,13 @@ std::vector track_impl::containing_crates() std::string track_impl::relative_path() { - const auto path = snapshot().relative_path; - return path.value_or(std::string{}); + const auto path = content_table{context_}.get_path(id()); + if (!path) + return {}; + + // Paths are absolute within the device, whereas djinterop wants them + // relative to the directory of the database. + return path->front() == '/' ? path->substr(1) : *path; } std::string track_impl::filename() @@ -90,12 +99,12 @@ std::string track_impl::file_extension() std::optional track_impl::album() { - return snapshot().album; + return content_table{context_}.get_album(id()); } std::optional track_impl::artist() { - return snapshot().artist; + return content_table{context_}.get_artist(id()); } std::optional track_impl::average_loudness() @@ -110,32 +119,44 @@ std::vector track_impl::beatgrid() std::optional track_impl::bitrate() { - return snapshot().bitrate; + const auto bitrate = content_table{context_}.get_bitrate(id()); + if (!bitrate || *bitrate <= 0) + return std::nullopt; + + return static_cast(*bitrate); } std::optional track_impl::bpm() { - return snapshot().bpm; + const auto bpm_x100 = content_table{context_}.get_bpm_x100(id()); + if (!bpm_x100 || *bpm_x100 <= 0) + return std::nullopt; + + return static_cast(*bpm_x100) / 100; } std::optional track_impl::comment() { - return snapshot().comment; + return content_table{context_}.get_comment(id()); } std::optional track_impl::composer() { - return snapshot().composer; + return content_table{context_}.get_composer(id()); } std::optional track_impl::duration() { - return snapshot().duration; + const auto length = content_table{context_}.get_length(id()); + if (!length || length->count() <= 0) + return std::nullopt; + + return std::chrono::duration_cast(*length); } std::optional track_impl::genre() { - return snapshot().genre; + return content_table{context_}.get_genre(id()); } std::optional track_impl::hot_cue_at(int) @@ -150,7 +171,11 @@ std::vector> track_impl::hot_cues() std::optional track_impl::key() { - return snapshot().key; + const auto name = content_table{context_}.get_key(id()); + if (!name) + return std::nullopt; + + return parse_musical_key(*name); } std::optional @@ -177,32 +202,54 @@ std::optional track_impl::main_cue() std::optional track_impl::publisher() { - return snapshot().publisher; + return content_table{context_}.get_label(id()); } std::optional track_impl::rating() { - return snapshot().rating; + const auto stars = content_table{context_}.get_rating_stars(id()); + if (!stars) + return std::nullopt; + + // djinterop rates a track from zero to one hundred, where rekordbox uses + // whole stars. + return static_cast(std::clamp(*stars, 0, 5) * 20); } std::optional track_impl::sample_count() { - return snapshot().sample_count; + // The database records a duration in whole seconds and no sample count, + // so the count can only be recovered to that precision. + const content_table content{context_}; + const auto length = content.get_length(id()); + const auto rate = content.get_sampling_rate(id()); + if (!length || length->count() <= 0 || !rate || *rate <= 0) + return std::nullopt; + + return static_cast(length->count() * *rate); } std::optional track_impl::sample_rate() { - return snapshot().sample_rate; + const auto rate = content_table{context_}.get_sampling_rate(id()); + if (!rate || *rate <= 0) + return std::nullopt; + + return static_cast(*rate); } std::optional track_impl::title() { - return snapshot().title; + return content_table{context_}.get_title(id()); } std::optional track_impl::track_number() { - return snapshot().track_number; + const auto number = content_table{context_}.get_track_number(id()); + if (!number || *number <= 0) + return std::nullopt; + + return static_cast(*number); } std::vector track_impl::waveform() @@ -212,7 +259,11 @@ std::vector track_impl::waveform() std::optional track_impl::year() { - return snapshot().year; + const auto year = content_table{context_}.get_release_year(id()); + if (!year || *year <= 0) + return std::nullopt; + + return static_cast(*year); } void track_impl::update(const track_snapshot&) @@ -351,4 +402,4 @@ void track_impl::set_year(std::optional) read_only(); } -} // namespace djinterop::onelibrary +} // namespace djinterop::onelibrary::v1 diff --git a/src/djinterop/onelibrary/track_impl.hpp b/src/djinterop/onelibrary/v1/track_impl.hpp similarity index 94% rename from src/djinterop/onelibrary/track_impl.hpp rename to src/djinterop/onelibrary/v1/track_impl.hpp index 57a2968..3cd500c 100644 --- a/src/djinterop/onelibrary/track_impl.hpp +++ b/src/djinterop/onelibrary/v1/track_impl.hpp @@ -18,6 +18,7 @@ #pragma once #include + #include #include #include @@ -27,10 +28,9 @@ #include #include -#include "../impl/track_impl.hpp" -#include "onelibrary_context.hpp" - -namespace djinterop::onelibrary +#include "../../impl/track_impl.hpp" +#include "../onelibrary_context.hpp" +namespace djinterop::onelibrary::v1 { /// A track in a OneLibrary database. /// @@ -79,8 +79,9 @@ class track_impl : public djinterop::track_impl void set_key(std::optional key) override; std::optional last_played_at() override; - void set_last_played_at(std::optional - played_at) override; + void set_last_played_at( + std::optional played_at) + override; std::optional loop_at(int index) override; void set_loop_at(int index, std::optional l) override; std::vector> loops() override; @@ -111,4 +112,4 @@ class track_impl : public djinterop::track_impl std::shared_ptr context_; }; -} // namespace djinterop::onelibrary +} // namespace djinterop::onelibrary::v1 diff --git a/src/djinterop/util/crypto/encrypted_database_builtin.cpp b/src/djinterop/util/crypto/encrypted_database_builtin.cpp index f53355b..93e43ca 100644 --- a/src/djinterop/util/crypto/encrypted_database_builtin.cpp +++ b/src/djinterop/util/crypto/encrypted_database_builtin.cpp @@ -51,9 +51,11 @@ sqlite::database open_encrypted_database( #if !DJINTEROP_HAVE_DESERIALIZE (void)passphrase; throw encryption_unsupported{ - "The database `" + path + - "` needs SQLite 3.36 or newer, built without SQLITE_OMIT_DESERIALIZE, " - "to read"}; + "This build of libdjinterop cannot read the encrypted database `" + + path + + "`. Reading one goes through `sqlite3_deserialize`, which needs " + "SQLite 3.36 or newer, built without SQLITE_OMIT_DESERIALIZE, and " + "this build was made against an older or narrower SQLite."}; #else // Key derivation is expensive, so it is done once here. const auto codec = make_codec_for(path, passphrase); diff --git a/src/djinterop/util/sqlite_query.hpp b/src/djinterop/util/sqlite_query.hpp new file mode 100644 index 0000000..506f644 --- /dev/null +++ b/src/djinterop/util/sqlite_query.hpp @@ -0,0 +1,59 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include +#include +#include + +#include + +namespace djinterop::util +{ +/// Run a query whose rows are each a single identifier. +template +std::vector collect_ids( + sqlite::database& db, const char* sql, const Args&... args) +{ + std::vector results; + auto query = db << sql; + ((query << args), ...); + query >> [&](int64_t id) { results.push_back(id); }; + return results; +} + +/// Run a query whose rows are each a single identifier, and take the first. +template +std::optional first_id( + sqlite::database& db, const char* sql, const Args&... args) +{ + std::optional result; + auto query = db << sql; + ((query << args), ...); + query >> [&](int64_t id) { result = id; }; + return result; +} + +/// Test whether a query matches any row at all. +template +bool any_row(sqlite::database& db, const char* sql, const Args&... args) +{ + return first_id(db, sql, args...).has_value(); +} + +} // namespace djinterop::util diff --git a/test/djinterop/onelibrary/content_table_test.cpp b/test/djinterop/onelibrary/content_table_test.cpp index 90ec9cd..e5cf835 100644 --- a/test/djinterop/onelibrary/content_table_test.cpp +++ b/test/djinterop/onelibrary/content_table_test.cpp @@ -18,6 +18,7 @@ #define BOOST_TEST_MODULE onelibrary_content_table_test #include +#include #include #include @@ -25,13 +26,15 @@ #include -#include "../../../src/djinterop/onelibrary/content_table.hpp" +#include #include "../../../src/djinterop/onelibrary/onelibrary_context.hpp" +#include "../../../src/djinterop/onelibrary/v1/track_conversion.hpp" #include "../boost_test_printable.hpp" #include "onelibrary_schema.hpp" namespace utf = boost::unit_test; namespace ol = djinterop::onelibrary; +namespace olv1 = djinterop::onelibrary::v1; namespace { @@ -74,7 +77,7 @@ BOOST_TEST_DECORATOR(*utf::description("get() resolves the lookup tables")) BOOST_AUTO_TEST_CASE(get__a_populated_row__resolves_its_lookups) { // Arrange - const ol::content_table content{make_context()}; + const olv1::content_table content{make_context()}; // Act const auto row = content.get(1); @@ -89,7 +92,7 @@ BOOST_AUTO_TEST_CASE(get__a_populated_row__resolves_its_lookups) BOOST_CHECK_EQUAL(row->label.value(), "Warp"); BOOST_CHECK_EQUAL(row->key.value(), "F#m"); BOOST_CHECK_EQUAL(row->bpm_x100.value(), 12400); - BOOST_CHECK_EQUAL(row->length_seconds.value(), 391); + BOOST_CHECK(row->length.value() == std::chrono::seconds{391}); BOOST_CHECK_EQUAL(row->rating_stars.value(), 4); BOOST_CHECK_EQUAL(row->path.value(), "/Contents/Aphex/alpha.mp3"); } @@ -98,7 +101,7 @@ BOOST_TEST_DECORATOR(*utf::description("get() reads an empty column as absent")) BOOST_AUTO_TEST_CASE(get__an_empty_column__reads_as_absent) { // Arrange - const ol::content_table content{make_context()}; + const olv1::content_table content{make_context()}; // Act const auto row = content.get(2); @@ -114,7 +117,7 @@ BOOST_TEST_DECORATOR(*utf::description("get() for a row that is not there")) BOOST_AUTO_TEST_CASE(get__an_unknown_row__is_absent) { // Arrange - const ol::content_table content{make_context()}; + const olv1::content_table content{make_context()}; // Act const auto row = content.get(404); @@ -127,7 +130,7 @@ BOOST_TEST_DECORATOR(*utf::description("all_ids() is ordered by identifier")) BOOST_AUTO_TEST_CASE(all_ids__a_populated_table__is_ordered) { // Arrange - const ol::content_table content{make_context()}; + const olv1::content_table content{make_context()}; // Act const auto ids = content.all_ids(); @@ -143,7 +146,7 @@ BOOST_TEST_DECORATOR( BOOST_AUTO_TEST_CASE(ids_by_path__either_spelling__finds_the_row) { // Arrange - const ol::content_table content{make_context()}; + const olv1::content_table content{make_context()}; // Act const auto absolute = content.ids_by_path("/Contents/Aphex/alpha.mp3"); @@ -159,7 +162,7 @@ BOOST_TEST_DECORATOR(*utf::description("exists() for present and absent rows")) BOOST_AUTO_TEST_CASE(exists__present_and_absent_rows__reports_each) { // Arrange - const ol::content_table content{make_context()}; + const olv1::content_table content{make_context()}; // Act / Assert BOOST_CHECK(content.exists(1)); @@ -171,12 +174,12 @@ BOOST_TEST_DECORATOR( BOOST_AUTO_TEST_CASE(to_snapshot__a_populated_row__converts_its_units) { // Arrange - const ol::content_table content{make_context()}; + const olv1::content_table content{make_context()}; const auto row = content.get(1); BOOST_REQUIRE(row); // Act - const auto snapshot = ol::to_snapshot(*row); + const auto snapshot = olv1::to_snapshot(*row); // Assert BOOST_CHECK_CLOSE(snapshot.bpm.value(), 124.0, 0.001); @@ -200,12 +203,12 @@ BOOST_TEST_DECORATOR( BOOST_AUTO_TEST_CASE(to_snapshot__any_row__has_no_performance_data) { // Arrange - const ol::content_table content{make_context()}; + const olv1::content_table content{make_context()}; const auto row = content.get(1); BOOST_REQUIRE(row); // Act - const auto snapshot = ol::to_snapshot(*row); + const auto snapshot = olv1::to_snapshot(*row); // Assert: rekordbox leaves these in the ANLZ files beside the database. BOOST_CHECK(snapshot.beatgrid.empty()); @@ -219,18 +222,18 @@ BOOST_TEST_DECORATOR( BOOST_AUTO_TEST_CASE(parse_musical_key__known_notations__are_understood) { // Act / Assert - BOOST_CHECK(ol::parse_musical_key("C") == djinterop::musical_key::c_major); - BOOST_CHECK(ol::parse_musical_key("Am") == djinterop::musical_key::a_minor); + BOOST_CHECK(olv1::parse_musical_key("C") == djinterop::musical_key::c_major); + BOOST_CHECK(olv1::parse_musical_key("Am") == djinterop::musical_key::a_minor); BOOST_CHECK( - ol::parse_musical_key("F#m") == djinterop::musical_key::f_sharp_minor); + olv1::parse_musical_key("F#m") == djinterop::musical_key::f_sharp_minor); BOOST_CHECK( - ol::parse_musical_key("Bb") == djinterop::musical_key::b_flat_major); + olv1::parse_musical_key("Bb") == djinterop::musical_key::b_flat_major); // The typographic accidentals mean the same as the ASCII ones. BOOST_CHECK( - ol::parse_musical_key("F♯m") == djinterop::musical_key::f_sharp_minor); + olv1::parse_musical_key("F♯m") == djinterop::musical_key::f_sharp_minor); BOOST_CHECK( - ol::parse_musical_key("B♭") == djinterop::musical_key::b_flat_major); + olv1::parse_musical_key("B♭") == djinterop::musical_key::b_flat_major); } BOOST_TEST_DECORATOR( @@ -238,10 +241,10 @@ BOOST_TEST_DECORATOR( BOOST_AUTO_TEST_CASE(parse_musical_key__unknown_notations__are_not_guessed) { // Act / Assert - BOOST_CHECK(!ol::parse_musical_key("")); - BOOST_CHECK(!ol::parse_musical_key("H")); + BOOST_CHECK(!olv1::parse_musical_key("")); + BOOST_CHECK(!olv1::parse_musical_key("H")); // The Camelot and Open Key wheels are not read. - BOOST_CHECK(!ol::parse_musical_key("8A")); - BOOST_CHECK(!ol::parse_musical_key("Camelot 8A")); + BOOST_CHECK(!olv1::parse_musical_key("8A")); + BOOST_CHECK(!olv1::parse_musical_key("Camelot 8A")); } diff --git a/test/djinterop/onelibrary/database_test.cpp b/test/djinterop/onelibrary/database_test.cpp index 84b2bbf..7271705 100644 --- a/test/djinterop/onelibrary/database_test.cpp +++ b/test/djinterop/onelibrary/database_test.cpp @@ -28,6 +28,7 @@ #include #include +#include #include "../../../src/djinterop/util/filesystem.hpp" #include "../boost_test_printable.hpp" @@ -37,6 +38,7 @@ namespace utf = boost::unit_test; namespace ol = djinterop::onelibrary; +namespace olv1 = djinterop::onelibrary::v1; namespace crypto = djinterop::util::crypto; namespace @@ -231,9 +233,9 @@ djinterop::database loaded_database() } /// The shared device, opened once as a library. -const ol::library& loaded_library() +const olv1::library& loaded_library() { - static const ol::library lib{device_fixture().path, passphrase}; + static const olv1::library lib{device_fixture().path, passphrase}; return lib; } @@ -511,6 +513,12 @@ BOOST_AUTO_TEST_CASE( prepare_plain_database(db); create_onelibrary_schema(db); + // Every export carries a `property` row, whatever else it holds. + execute( + db, + "INSERT INTO property VALUES ('FIXTURE', '1000', 0, " + "'2026-01-01', 0, 0)"); + // Root // +- Middle A (sequence 1) // | +- Leaf A (sequence 1) @@ -782,7 +790,7 @@ BOOST_AUTO_TEST_CASE(library__a_device__is_read_through_its_database) const auto& lib = loaded_library(); // Act - auto db = lib.db(); + auto db = lib.database(); // Assert BOOST_CHECK_EQUAL(lib.directory(), device_fixture().path); @@ -791,62 +799,76 @@ BOOST_AUTO_TEST_CASE(library__a_device__is_read_through_its_database) } BOOST_TEST_DECORATOR(*utf::description( - "library::analysis_path() gives a path relative to the device")) -BOOST_AUTO_TEST_CASE(analysis_path__a_track_with_analysis_data__is_relative) + "content_table::get_analysis_path() gives the path the device records")) +BOOST_AUTO_TEST_CASE(get_analysis_path__a_track_with_analysis_data__is_read) { // Arrange const auto& lib = loaded_library(); // Act - const auto path = lib.analysis_path(1); + const auto path = lib.content().get_analysis_path(1); // Assert BOOST_REQUIRE(path); - BOOST_CHECK_EQUAL(*path, "PIONEER/USBANLZ/P016/0000875e/ANLZ0000.DAT"); + BOOST_CHECK_EQUAL(*path, "/PIONEER/USBANLZ/P016/0000875e/ANLZ0000.DAT"); } BOOST_TEST_DECORATOR(*utf::description( - "library::analysis_path() for a track that carries none, and for one that " - "is not there")) -BOOST_AUTO_TEST_CASE(analysis_path__no_analysis_data__is_absent) + "content_table::get_analysis_path() for a track that carries none, and " + "for one that is not there")) +BOOST_AUTO_TEST_CASE(get_analysis_path__no_analysis_data__is_absent) { // Arrange const auto& lib = loaded_library(); // Act, Assert - BOOST_CHECK(!lib.analysis_path(2)); - BOOST_CHECK(!lib.analysis_path(1234)); + BOOST_CHECK(!lib.content().get_analysis_path(2)); + BOOST_CHECK(!lib.content().get_analysis_path(1234)); } BOOST_TEST_DECORATOR(*utf::description( - "library::key_name() gives back the notation the device holds")) -BOOST_AUTO_TEST_CASE(key_name__any_track__is_the_notation_on_the_device) + "content_table::get_key() gives back the notation the device holds")) +BOOST_AUTO_TEST_CASE(get_key__any_track__is_the_notation_on_the_device) { // Arrange const auto& lib = loaded_library(); // Act, Assert - BOOST_CHECK_EQUAL(lib.key_name(1).value(), "F#m"); - BOOST_CHECK_EQUAL(lib.key_name(2).value(), "Bb"); + BOOST_CHECK_EQUAL(lib.content().get_key(1).value(), "F#m"); + BOOST_CHECK_EQUAL(lib.content().get_key(2).value(), "Bb"); // A notation the library does not parse is still given back whole. - BOOST_CHECK_EQUAL(lib.key_name(3).value(), "Camelot 8A"); + BOOST_CHECK_EQUAL(lib.content().get_key(3).value(), "Camelot 8A"); - BOOST_CHECK(!lib.key_name(1234)); + BOOST_CHECK(!lib.content().get_key(1234)); } -BOOST_TEST_DECORATOR( - *utf::description("library::color_id() reads the colour of a track")) -BOOST_AUTO_TEST_CASE(color_id__marked_and_unmarked_tracks__reports_each) +BOOST_TEST_DECORATOR(*utf::description( + "content_table::get_color_id() reads the colour of a track")) +BOOST_AUTO_TEST_CASE(get_color_id__marked_and_unmarked_tracks__reports_each) { // Arrange const auto& lib = loaded_library(); // Act, Assert - BOOST_CHECK_EQUAL(lib.color_id(1).value(), 6); + BOOST_CHECK_EQUAL(lib.content().get_color_id(1).value(), 6); - // A colour of zero is no colour, as is a column that is not set at all. - BOOST_CHECK(!lib.color_id(2)); - BOOST_CHECK(!lib.color_id(3)); - BOOST_CHECK(!lib.color_id(1234)); + // The low-level API reports what the column holds, and `COLOR_ID_NONE` is + // what an unmarked track carries. + BOOST_CHECK_EQUAL( + lib.content().get_color_id(2).value(), olv1::COLOR_ID_NONE); + BOOST_CHECK(!lib.content().get_color_id(3)); + BOOST_CHECK(!lib.content().get_color_id(1234)); +} + +BOOST_TEST_DECORATOR(*utf::description( + "property_table::get_db_version() reports the schema version")) +BOOST_AUTO_TEST_CASE(get_db_version__a_device__is_the_version_it_records) +{ + // Arrange + const auto& lib = loaded_library(); + + // Act, Assert + BOOST_CHECK_EQUAL( + lib.property().get_db_version().value(), olv1::supported_db_version); } diff --git a/test/djinterop/onelibrary/onelibrary_schema.hpp b/test/djinterop/onelibrary/onelibrary_schema.hpp index f2bb1e3..447b535 100644 --- a/test/djinterop/onelibrary/onelibrary_schema.hpp +++ b/test/djinterop/onelibrary/onelibrary_schema.hpp @@ -17,59 +17,64 @@ #pragma once +#include +#include +#include #include #include -/// The schema a real OneLibrary export carries, abbreviated to the tables that -/// this library reads. -/// -/// Nothing in a real export is declared `NOT NULL`, and no foreign key is -/// enforced, so the fixtures can leave any column unset. +#define ONELIBRARY_STRINGIFY(x) ONELIBRARY_STRINGIFY_(x) +#define ONELIBRARY_STRINGIFY_(x) #x + +/// The schema a real OneLibrary export carries, read from the reference +/// script in `testdata/ref/onelibrary`. /// -/// The published description of the format, which the pyrekordbox project -/// documents as Device Library Plus, lists neither `content.djPlayCount` nor -/// `album.image_id`, and gives `playlist_content` and `property` primary keys -/// that an export does not carry. What a device holds is what is written -/// here. +/// The script is the record of what a device holds; see its own comments. inline const std::vector& onelibrary_schema_statements() { - static const std::vector statements{ - "CREATE TABLE content(content_id integer primary key, title varchar, " - "titleForSearch varchar, subtitle varchar, bpmx100 integer, " - "length integer, trackNo integer, discNo integer, " - "artist_id_artist integer, artist_id_remixer integer, " - "artist_id_originalArtist integer, artist_id_composer integer, " - "artist_id_lyricist integer, album_id integer, genre_id integer, " - "label_id integer, key_id integer, color_id integer, " - "image_id integer, djComment varchar, rating integer, " - "releaseYear integer, releaseDate varchar, dateCreated varchar, " - "dateAdded varchar, path varchar, fileName varchar, " - "fileSize integer, fileType integer, bitrate integer, " - "bitDepth integer, samplingRate integer, isrc varchar, " - "djPlayCount integer, isHotCueAutoLoadOn integer, " - "isKuvoDeliverStatusOn integer, kuvoDeliveryComment varchar, " - "masterDbId integer, masterContentId integer, " - "analysisDataFilePath varchar, analysedBits integer, " - "contentLink integer, hasModified integer, cueUpdateCount integer, " - "analysisDataUpdateCount integer, informationUpdateCount integer)", - "CREATE TABLE artist(artist_id integer primary key, " - "name varchar, nameForSearch varchar)", - "CREATE TABLE album(album_id integer primary key, " - "name varchar, artist_id integer, image_id integer, " - "isComplation integer, nameForSearch varchar)", - "CREATE TABLE genre(genre_id integer primary key, name varchar)", - "CREATE TABLE label(label_id integer primary key, name varchar)", - "CREATE TABLE \"key\"(key_id integer primary key, name varchar)", - "CREATE TABLE playlist(playlist_id integer primary key, " - "sequenceNo integer, name varchar, image_id integer, " - "attribute integer, playlist_id_parent integer)", - "CREATE TABLE playlist_content(playlist_id integer, " - "content_id integer, sequenceNo integer)", - "CREATE TABLE property(deviceName varchar, " - "dbVersion varchar, numberOfContents integer, " - "createdDate varchar, backGroundColorType integer, " - "myTagMasterDBID integer)", - }; + static const std::vector statements = [] + { + const std::string path = + std::string{ONELIBRARY_STRINGIFY(TESTDATA_DIR)} + + "/ref/onelibrary/schema.sql"; + + std::ifstream file{path}; + if (!file) + throw std::runtime_error{"Cannot read the schema at " + path}; + + std::ostringstream contents; + contents << file.rdbuf(); + + // The script is a sequence of statements separated by semicolons, and + // nothing in it holds one in a string literal. + std::vector result; + std::string statement; + for (const auto character : contents.str()) + { + if (character != ';') + { + statement += character; + continue; + } + + // Comment lines belong to the script, not to the statement. + std::string stripped; + std::istringstream lines{statement}; + for (std::string line; std::getline(lines, line);) + if (line.rfind("--", 0) != 0) + stripped += line + " "; + + const auto begin = stripped.find_first_not_of(" \t\r\n"); + const auto end = stripped.find_last_not_of(" \t\r\n"); + if (begin != std::string::npos) + result.push_back( + stripped.substr(begin, end - begin + 1)); + + statement.clear(); + } + + return result; + }(); return statements; } diff --git a/test/djinterop/onelibrary/playlist_table_test.cpp b/test/djinterop/onelibrary/playlist_table_test.cpp index 7b4f445..8d30c53 100644 --- a/test/djinterop/onelibrary/playlist_table_test.cpp +++ b/test/djinterop/onelibrary/playlist_table_test.cpp @@ -24,12 +24,13 @@ #include #include "../../../src/djinterop/onelibrary/onelibrary_context.hpp" -#include "../../../src/djinterop/onelibrary/playlist_table.hpp" +#include #include "../boost_test_printable.hpp" #include "onelibrary_schema.hpp" namespace utf = boost::unit_test; namespace ol = djinterop::onelibrary; +namespace olv1 = djinterop::onelibrary::v1; namespace { @@ -69,7 +70,7 @@ BOOST_TEST_DECORATOR(*utf::description("get() reads one row of the tree")) BOOST_AUTO_TEST_CASE(get__a_child__reads_its_row) { // Arrange - const ol::playlist_table playlists{make_context()}; + const olv1::playlist_table playlists{make_context()}; // Act const auto row = playlists.get(2); @@ -88,7 +89,7 @@ BOOST_TEST_DECORATOR( BOOST_AUTO_TEST_CASE(get__a_parent_of_zero__reads_as_no_parent) { // Arrange - const ol::playlist_table playlists{make_context()}; + const olv1::playlist_table playlists{make_context()}; // Act const auto row = playlists.get(5); @@ -102,7 +103,7 @@ BOOST_TEST_DECORATOR(*utf::description("get() for a row that is not there")) BOOST_AUTO_TEST_CASE(get__an_unknown_playlist__is_absent) { // Arrange - const ol::playlist_table playlists{make_context()}; + const olv1::playlist_table playlists{make_context()}; // Act / Assert BOOST_CHECK(!playlists.get(404)); @@ -115,7 +116,7 @@ BOOST_TEST_DECORATOR( BOOST_AUTO_TEST_CASE(root_ids__both_spellings__are_roots) { // Arrange - const ol::playlist_table playlists{make_context()}; + const olv1::playlist_table playlists{make_context()}; // Act const auto ids = playlists.root_ids(); @@ -130,7 +131,7 @@ BOOST_TEST_DECORATOR(*utf::description("child_ids() is in sibling order")) BOOST_AUTO_TEST_CASE(child_ids__several_siblings__are_in_sequence) { // Arrange - const ol::playlist_table playlists{make_context()}; + const olv1::playlist_table playlists{make_context()}; // Act const auto ids = playlists.child_ids(1); @@ -145,7 +146,7 @@ BOOST_TEST_DECORATOR(*utf::description("descendant_ids() is breadth first")) BOOST_AUTO_TEST_CASE(descendant_ids__a_deep_tree__is_breadth_first) { // Arrange - const ol::playlist_table playlists{make_context()}; + const olv1::playlist_table playlists{make_context()}; // Act const auto ids = playlists.descendant_ids(1); @@ -162,7 +163,7 @@ BOOST_TEST_DECORATOR( BOOST_AUTO_TEST_CASE(find__a_known_name__is_found_at_its_own_level) { // Arrange - const ol::playlist_table playlists{make_context()}; + const olv1::playlist_table playlists{make_context()}; // Act / Assert BOOST_CHECK(playlists.find_root("Sets") == 1); @@ -177,7 +178,7 @@ BOOST_TEST_DECORATOR(*utf::description("track_ids() is in playlist order")) BOOST_AUTO_TEST_CASE(track_ids__a_populated_playlist__is_in_order) { // Arrange - const ol::playlist_table playlists{make_context()}; + const olv1::playlist_table playlists{make_context()}; // Act const auto ids = playlists.track_ids(2); @@ -193,7 +194,7 @@ BOOST_TEST_DECORATOR( BOOST_AUTO_TEST_CASE(playlists_containing__a_shared_track__finds_each_holder) { // Arrange - const ol::playlist_table playlists{make_context()}; + const olv1::playlist_table playlists{make_context()}; // Act const auto holders = playlists.playlists_containing(10); diff --git a/testdata/ref/onelibrary/schema.sql b/testdata/ref/onelibrary/schema.sql new file mode 100644 index 0000000..6bbb6dd --- /dev/null +++ b/testdata/ref/onelibrary/schema.sql @@ -0,0 +1,29 @@ +-- The schema a real OneLibrary export carries, abbreviated to the tables that +-- libdjinterop reads. +-- +-- Nothing in a real export is declared NOT NULL, and no foreign key is +-- enforced, so a fixture can leave any column unset. +-- +-- The published description of the format, which the pyrekordbox project +-- documents as Device Library Plus, lists neither content.djPlayCount nor +-- album.image_id, and gives playlist_content and property primary keys that an +-- export does not carry. What a device holds is what is written here. + +CREATE TABLE content(content_id integer primary key, title varchar, titleForSearch varchar, subtitle varchar, bpmx100 integer, length integer, trackNo integer, discNo integer, artist_id_artist integer, artist_id_remixer integer, artist_id_originalArtist integer, artist_id_composer integer, artist_id_lyricist integer, album_id integer, genre_id integer, label_id integer, key_id integer, color_id integer, image_id integer, djComment varchar, rating integer, releaseYear integer, releaseDate varchar, dateCreated varchar, dateAdded varchar, path varchar, fileName varchar, fileSize integer, fileType integer, bitrate integer, bitDepth integer, samplingRate integer, isrc varchar, djPlayCount integer, isHotCueAutoLoadOn integer, isKuvoDeliverStatusOn integer, kuvoDeliveryComment varchar, masterDbId integer, masterContentId integer, analysisDataFilePath varchar, analysedBits integer, contentLink integer, hasModified integer, cueUpdateCount integer, analysisDataUpdateCount integer, informationUpdateCount integer); + +CREATE TABLE artist(artist_id integer primary key, name varchar, nameForSearch varchar); + +CREATE TABLE album(album_id integer primary key, name varchar, artist_id integer, image_id integer, isComplation integer, nameForSearch varchar); + +CREATE TABLE genre(genre_id integer primary key, name varchar); + +CREATE TABLE label(label_id integer primary key, name varchar); + +CREATE TABLE "key"(key_id integer primary key, name varchar); + +CREATE TABLE playlist(playlist_id integer primary key, sequenceNo integer, name varchar, image_id integer, attribute integer, playlist_id_parent integer); + +CREATE TABLE playlist_content(playlist_id integer, content_id integer, sequenceNo integer); + +CREATE TABLE property(deviceName varchar, dbVersion varchar, numberOfContents integer, createdDate varchar, backGroundColorType integer, myTagMasterDBID integer); + From 42eaa5ed867674550e6d412c19ffe12437ce2c65 Mon Sep 17 00:00:00 2001 From: Illia Komsa Date: Wed, 23 Sep 2026 12:35:26 +0200 Subject: [PATCH 09/10] Read OneLibrary databases through SQLCipher Main now vendors SQLCipher, so the built-in decryption goes: the AES, SHA-512, page codec and WAL folding, with their tests and CMake checks. A OneLibrary database is opened in place with SQLCipher when built with EXPERIMENTAL_ENABLE_SQLCIPHER, and refused with an explanation otherwise. The OneLibrary tests build on that switch and write their fixtures with SQLCipher itself. Also simplify along the way: share the track field conversions between snapshots and accessors, check the schema beside the loader, drop the loader header and the internal exception types, and resolve the device root with std::filesystem. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 1 - CMakeLists.txt | 209 ++------- DjInteropConfig.cmake.in | 2 - GUIDE.md | 12 +- README.md | 2 + example/onelibrary.cpp | 18 +- include/djinterop/onelibrary/onelibrary.hpp | 8 +- include/djinterop/onelibrary/v1/library.hpp | 18 +- .../onelibrary/v1/playlist_table.hpp | 9 +- src/djinterop/onelibrary/onelibrary.cpp | 99 ++--- .../onelibrary/onelibrary_context.hpp | 14 +- src/djinterop/onelibrary/v1/content_table.cpp | 6 +- src/djinterop/onelibrary/v1/crate_impl.cpp | 3 +- src/djinterop/onelibrary/v1/database_impl.cpp | 30 +- src/djinterop/onelibrary/v1/library.cpp | 20 +- src/djinterop/onelibrary/v1/playlist_impl.cpp | 3 +- src/djinterop/onelibrary/v1/playlist_impl.hpp | 1 - .../onelibrary/v1/track_conversion.cpp | 135 +++--- .../onelibrary/v1/track_conversion.hpp | 31 +- src/djinterop/onelibrary/v1/track_impl.cpp | 69 +-- src/djinterop/onelibrary/v1/track_impl.hpp | 3 + src/djinterop/util/crypto/aes.cpp | 353 ---------------- src/djinterop/util/crypto/aes.hpp | 98 ----- src/djinterop/util/crypto/aes_hardware.cpp | 321 -------------- src/djinterop/util/crypto/aes_hardware.hpp | 53 --- .../util/crypto/encrypted_database.hpp | 65 --- .../crypto/encrypted_database_builtin.cpp | 107 ----- src/djinterop/util/crypto/sha512.cpp | 336 --------------- src/djinterop/util/crypto/sha512.hpp | 99 ----- src/djinterop/util/crypto/sqlcipher_codec.cpp | 203 --------- src/djinterop/util/crypto/sqlcipher_codec.hpp | 161 ------- src/djinterop/util/crypto/sqlcipher_wal.cpp | 349 --------------- src/djinterop/util/crypto/sqlcipher_wal.hpp | 51 --- src/djinterop/util/filesystem.cpp | 9 - src/djinterop/util/filesystem.hpp | 1 - src/djinterop/util/sqlcipher.cpp | 58 +++ src/djinterop/util/sqlcipher.hpp | 41 ++ .../sqlcipher_unsupported.cpp} | 23 +- src/djinterop/util/sqlite_query.hpp | 10 +- .../onelibrary/content_table_test.cpp | 3 +- test/djinterop/onelibrary/database_test.cpp | 299 ++----------- .../onelibrary/onelibrary_schema.hpp | 69 +-- .../onelibrary/playlist_table_test.cpp | 3 +- test/djinterop/sqlcipher_encryptor.hpp | 134 ------ test/djinterop/util/crypto_test.cpp | 398 ------------------ 45 files changed, 457 insertions(+), 3480 deletions(-) delete mode 100644 src/djinterop/util/crypto/aes.cpp delete mode 100644 src/djinterop/util/crypto/aes.hpp delete mode 100644 src/djinterop/util/crypto/aes_hardware.cpp delete mode 100644 src/djinterop/util/crypto/aes_hardware.hpp delete mode 100644 src/djinterop/util/crypto/encrypted_database.hpp delete mode 100644 src/djinterop/util/crypto/encrypted_database_builtin.cpp delete mode 100644 src/djinterop/util/crypto/sha512.cpp delete mode 100644 src/djinterop/util/crypto/sha512.hpp delete mode 100644 src/djinterop/util/crypto/sqlcipher_codec.cpp delete mode 100644 src/djinterop/util/crypto/sqlcipher_codec.hpp delete mode 100644 src/djinterop/util/crypto/sqlcipher_wal.cpp delete mode 100644 src/djinterop/util/crypto/sqlcipher_wal.hpp create mode 100644 src/djinterop/util/sqlcipher.cpp create mode 100644 src/djinterop/util/sqlcipher.hpp rename src/djinterop/{onelibrary/loader.hpp => util/sqlcipher_unsupported.cpp} (64%) delete mode 100644 test/djinterop/sqlcipher_encryptor.hpp delete mode 100644 test/djinterop/util/crypto_test.cpp diff --git a/.gitignore b/.gitignore index a38f9fc..1e8863e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,3 @@ compile_commands.json # CMake typical build dirs /cmake_build* /cmake-build* - diff --git a/CMakeLists.txt b/CMakeLists.txt index a5463cd..91ccbc3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,82 +50,6 @@ option(SYSTEM_SQLITE "Use system installation of SQLite" ON) option(EXPERIMENTAL_ENABLE_SQLCIPHER "Use SQLCipher in place of SQLite" OFF) option(SYSTEM_SQLITE_MODERN_CPP "Use system installation of sqlite_modern_cpp" OFF) - -# The processor's own AES instructions are worth two orders of magnitude over -# software AES when reading an encrypted OneLibrary database, so use them where -# the compiler can be persuaded to emit them. Whether the processor running the -# built library has them is a separate question, decided at run time; settled -# here is only whether the intrinsics compile, and with which flags. They are -# applied to `aes_hardware.cpp` alone, so no other translation unit can pick up -# an instruction the target may not have. -include(CheckCXXSourceCompiles) - -set(DJINTEROP_AES_X86_SOURCE " -#if defined(_MSC_VER) -#include -#else -#include -#endif -int main() -{ - __m128i x = _mm_setzero_si128(); - x = _mm_aesenc_si128(x, x); - x = _mm_aesenclast_si128(x, x); - x = _mm_aesdec_si128(x, x); - x = _mm_aesdeclast_si128(x, x); - return _mm_cvtsi128_si32(x); -} -") - -set(DJINTEROP_AES_ARM64_SOURCE " -#include -int main() -{ - uint8x16_t x = vdupq_n_u8(0); - x = vaesmcq_u8(vaeseq_u8(x, x)); - x = vaesimcq_u8(vaesdq_u8(x, x)); - return vgetq_lane_u8(x, 0); -} -") - -set(DJINTEROP_AES_INTRINSICS_DEFINE "") -set(DJINTEROP_AES_INTRINSICS_FLAGS "") - -# Try one architecture, unless another has already answered. Bare comes first: -# MSVC needs no flags at all, and neither does a compiler whose default target -# already includes the extension, such as Apple's on arm64. -macro(djinterop_try_aes arch source flags) - if(NOT DJINTEROP_AES_INTRINSICS_DEFINE) - check_cxx_source_compiles("${source}" DJINTEROP_AES_${arch}_BARE) - if(NOT DJINTEROP_AES_${arch}_BARE) - set(CMAKE_REQUIRED_FLAGS "${flags}") - check_cxx_source_compiles( - "${source}" DJINTEROP_AES_${arch}_FLAGGED) - unset(CMAKE_REQUIRED_FLAGS) - if(DJINTEROP_AES_${arch}_FLAGGED) - set(DJINTEROP_AES_INTRINSICS_FLAGS "${flags}") - endif() - endif() - if(DJINTEROP_AES_${arch}_BARE OR DJINTEROP_AES_${arch}_FLAGGED) - set(DJINTEROP_AES_INTRINSICS_DEFINE - "DJINTEROP_AES_INTRINSICS_${arch}") - endif() - endif() -endmacro() - -djinterop_try_aes(X86 "${DJINTEROP_AES_X86_SOURCE}" "-maes -msse2") -djinterop_try_aes(ARM64 "${DJINTEROP_AES_ARM64_SOURCE}" "-march=armv8-a+crypto") - -if(DJINTEROP_AES_INTRINSICS_DEFINE) - message(STATUS "Using AES instructions where available at run time") - set_source_files_properties( - src/djinterop/util/crypto/aes_hardware.cpp PROPERTIES - COMPILE_DEFINITIONS "${DJINTEROP_AES_INTRINSICS_DEFINE}" - COMPILE_FLAGS "${DJINTEROP_AES_INTRINSICS_FLAGS}") -else() - message(STATUS "No AES instructions for this target; using AES tables") -endif() - add_library( DjInterop include/djinterop/album_art.hpp @@ -280,7 +204,6 @@ add_library( src/djinterop/impl/playlist_impl.hpp src/djinterop/impl/track_impl.cpp src/djinterop/impl/track_impl.hpp - src/djinterop/onelibrary/loader.hpp src/djinterop/onelibrary/onelibrary.cpp src/djinterop/onelibrary/onelibrary_context.hpp src/djinterop/onelibrary/v1/content_table.cpp @@ -301,28 +224,23 @@ add_library( src/djinterop/track.cpp src/djinterop/util/chrono.cpp src/djinterop/util/chrono.hpp - src/djinterop/util/crypto/aes.cpp - src/djinterop/util/crypto/aes.hpp - src/djinterop/util/crypto/aes_hardware.cpp - src/djinterop/util/crypto/aes_hardware.hpp - src/djinterop/util/crypto/encrypted_database.hpp - # The one implementation of `encrypted_database.hpp`; a SQLCipher-backed - # build would name its own here instead. - src/djinterop/util/crypto/encrypted_database_builtin.cpp - src/djinterop/util/crypto/sha512.cpp - src/djinterop/util/crypto/sha512.hpp - src/djinterop/util/crypto/sqlcipher_codec.cpp - src/djinterop/util/crypto/sqlcipher_codec.hpp - src/djinterop/util/crypto/sqlcipher_wal.cpp - src/djinterop/util/crypto/sqlcipher_wal.hpp src/djinterop/util/filesystem.cpp src/djinterop/util/filesystem.hpp src/djinterop/util/random.cpp src/djinterop/util/random.hpp + src/djinterop/util/sqlcipher.hpp src/djinterop/util/sqlite_query.hpp src/djinterop/util/sqlite_transaction.hpp ) +# Encrypted OneLibrary databases are opened with SQLCipher, where the build has +# it. Otherwise, opening one fails with an explanation. +if(EXPERIMENTAL_ENABLE_SQLCIPHER) + target_sources(DjInterop PRIVATE src/djinterop/util/sqlcipher.cpp) +else() + target_sources(DjInterop PRIVATE src/djinterop/util/sqlcipher_unsupported.cpp) +endif() + set_target_properties(DjInterop PROPERTIES OUTPUT_NAME "djinterop" VERSION ${PROJECT_VERSION} @@ -349,14 +267,6 @@ target_include_directories( $ $) -# Decrypting the pages of a database is spread over the processors available, -# which is what the standard threading library is needed for. -set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(Threads REQUIRED) -target_link_libraries( - DjInterop PRIVATE - Threads::Threads) - # Always rely on system installation of zlib. set(ZLIB_MIN_VERSION 1.2.8) find_package(ZLIB ${ZLIB_MIN_VERSION} REQUIRED) @@ -403,7 +313,6 @@ elseif(SYSTEM_SQLITE) target_link_libraries( DjInterop PUBLIC ${SQLite3_LIBRARIES}) - set(DJINTEROP_SQLITE_VERSION "${SQLite3_VERSION}") else() # Use bundled SQLite amalgamation sources. message(STATUS "Using bundled SQLite...") @@ -416,16 +325,6 @@ else() target_include_directories( DjInterop PRIVATE SYSTEM ext/sqlite-amalgamation) - - # Read the version out of the amalgamation rather than restating it here, - # so that bumping the bundled copy is a matter of replacing two files. - file( - STRINGS ext/sqlite-amalgamation/sqlite3.h DJINTEROP_SQLITE_VERSION - REGEX "^#define SQLITE_VERSION[ \t]+\"") - string( - REGEX REPLACE "^#define SQLITE_VERSION[ \t]+\"([^\"]+)\".*" "\\1" - DJINTEROP_SQLITE_VERSION "${DJINTEROP_SQLITE_VERSION}") - message(STATUS "Bundled SQLite is version ${DJINTEROP_SQLITE_VERSION}") endif() if(SYSTEM_SQLITE_MODERN_CPP) @@ -584,10 +483,6 @@ if (Boost_FOUND AND Boost_filesystem_FOUND AND Boost_system_FOUND) ${Boost_INCLUDE_DIRS} ${CMAKE_CURRENT_BINARY_DIR}/include include) - # A test that compiles library sources into itself, rather than only - # linking, needs the headers that those sources include. - target_include_directories(${test_executable_name} PRIVATE SYSTEM - $) target_link_libraries(${test_executable_name} PUBLIC DjInterop ${Boost_LIBRARIES}) @@ -609,64 +504,34 @@ if (Boost_FOUND AND Boost_filesystem_FOUND AND Boost_system_FOUND) add_djinterop_test(engine/v3/ performance_data_table_test) add_djinterop_test(engine/v3/ track_table_test) - # Some parts of the library are internal, and its symbols are hidden, so - # tests of them compile those sources into themselves rather than linking. - # - # The OneLibrary tests are commented out below. Each of them talks to - # SQLite directly, and a test cannot borrow SQLite from the library: its - # symbols are hidden there. With `-DSYSTEM_SQLITE=OFF` that means every - # one of them compiles the amalgamation into itself, some fifteen seconds - # apiece, on top of the copy the library already builds. - # - # The sources are still in `test/djinterop/onelibrary`; uncommenting the - # block below is all it takes to run them again. `sqlite3_deserialize`, - # which reading a decrypted database goes through, arrived in SQLite 3.36, - # hence the version the database test asks for. - # - # Both problems go away once the format can be written: a fixture would - # then be built from `testdata/ref/onelibrary/schema.sql` through the - # library, as the Engine tests build one through - # `create_database_from_scripts()`, and no test would touch SQLite at all. - # - # if(SYSTEM_SQLITE) - # set(DJINTEROP_TEST_SQLITE_SOURCES "") - # else() - # set(DJINTEROP_TEST_SQLITE_SOURCES ext/sqlite-amalgamation/sqlite3.c) - # endif() - # - # add_djinterop_test(onelibrary/ content_table_test) - # target_sources(onelibrary_content_table_test PRIVATE - # src/djinterop/onelibrary/v1/content_table.cpp - # src/djinterop/onelibrary/v1/track_conversion.cpp - # ${DJINTEROP_TEST_SQLITE_SOURCES}) - # - # if(NOT DJINTEROP_SQLITE_VERSION VERSION_LESS 3.36) - # add_djinterop_test(onelibrary/ database_test) - # target_sources(onelibrary_database_test PRIVATE - # src/djinterop/util/crypto/aes.cpp - # src/djinterop/util/crypto/aes_hardware.cpp - # src/djinterop/util/crypto/sha512.cpp - # src/djinterop/util/crypto/sqlcipher_codec.cpp - # src/djinterop/util/filesystem.cpp - # ${DJINTEROP_TEST_SQLITE_SOURCES}) - # else() - # message( - # STATUS - # "OneLibrary database test not available, as reading the format " - # "needs SQLite 3.36 or newer") - # endif() - # - # add_djinterop_test(onelibrary/ playlist_table_test) - # target_sources(onelibrary_playlist_table_test PRIVATE - # src/djinterop/onelibrary/v1/playlist_table.cpp - # ${DJINTEROP_TEST_SQLITE_SOURCES}) - - add_djinterop_test(util/ crypto_test) - target_sources(util_crypto_test PRIVATE - src/djinterop/util/crypto/aes.cpp - src/djinterop/util/crypto/aes_hardware.cpp - src/djinterop/util/crypto/sha512.cpp - src/djinterop/util/crypto/sqlcipher_codec.cpp) + # The OneLibrary tests talk to SQLite directly to build their fixtures, and + # the library's own copy is hidden from them, so they link the SQLCipher + # static library themselves. For the same reason, they compile the + # internal sources they test into themselves rather than linking. + if(EXPERIMENTAL_ENABLE_SQLCIPHER) + add_djinterop_test(onelibrary/ content_table_test) + target_sources(onelibrary_content_table_test PRIVATE + src/djinterop/onelibrary/v1/content_table.cpp + src/djinterop/onelibrary/v1/track_conversion.cpp) + + add_djinterop_test(onelibrary/ database_test) + + add_djinterop_test(onelibrary/ playlist_table_test) + target_sources(onelibrary_playlist_table_test PRIVATE + src/djinterop/onelibrary/v1/playlist_table.cpp) + + foreach(test_name content_table_test database_test playlist_table_test) + target_link_libraries(onelibrary_${test_name} PRIVATE sqlcipher) + # The library sources compiled in need the headers they include. + target_include_directories(onelibrary_${test_name} PRIVATE SYSTEM + $) + endforeach() + else() + message( + STATUS + "OneLibrary tests not available, as reading the format needs " + "EXPERIMENTAL_ENABLE_SQLCIPHER") + endif() else() message(STATUS "Unit tests not available, as the Boost.Filesystem and Boost.System " diff --git a/DjInteropConfig.cmake.in b/DjInteropConfig.cmake.in index a18c8ce..22d88cf 100644 --- a/DjInteropConfig.cmake.in +++ b/DjInteropConfig.cmake.in @@ -8,8 +8,6 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") include(CMakeFindDependencyMacro) find_dependency(ZLIB) -set(THREADS_PREFER_PTHREAD_FLAG ON) -find_dependency(Threads) if(DJINTEROP_SYSTEM_DATE_H) find_dependency(date) endif() diff --git a/GUIDE.md b/GUIDE.md index 5eb28a6..a7c2209 100644 --- a/GUIDE.md +++ b/GUIDE.md @@ -102,10 +102,14 @@ aspects of the format are worth noting: * The format has a single tree that serves as both playlists and crates, so `playlists_and_crates_are_distinct` is false and the two views show the same rows. -* A device is read by decrypting it into memory. rekordbox writes the library - in write-ahead-logged mode, and the log has to be folded in before SQLite - sees the file, so reading one needs SQLite 3.36 or newer, built without - `SQLITE_OMIT_DESERIALIZE`. +* The database is encrypted with SQLCipher, so reading one needs libdjinterop + built with `-DEXPERIMENTAL_ENABLE_SQLCIPHER=ON -DSYSTEM_SQLITE=OFF`, and + OpenSSL. Without it, `load_database()` throws + `djinterop::unsupported_database`. +* rekordbox writes the database in write-ahead-logged mode, and SQLite cannot + read one of those without writing beside it, so reading a device creates + `-shm` and `-wal` files next to `exportLibrary.db` if they are missing. The + device therefore has to be writable. ### OneLibrary low-level API diff --git a/README.md b/README.md index b90219d..e275b99 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,8 @@ CDJ-3000 from firmware 3.15. Track metadata, playlists and crates can be read. Writing is not supported yet, and beat grids, waveforms, hot cues and loops are not held in the database at all, as rekordbox leaves them in the ANLZ files beside it. The format is also documented under the name Device Library Plus. +The database is encrypted with SQLCipher, so reading it needs the library built +with `-DEXPERIMENTAL_ENABLE_SQLCIPHER=ON -DSYSTEM_SQLITE=OFF`, and OpenSSL. What is not supported (yet): diff --git a/example/onelibrary.cpp b/example/onelibrary.cpp index 0ffb9ab..4eaf0e6 100644 --- a/example/onelibrary.cpp +++ b/example/onelibrary.cpp @@ -70,17 +70,21 @@ int main(int argc, char** argv) std::cout << "Tracks\n------\n"; for (auto&& track : db.tracks()) { + // A snapshot reads the whole track at once, where each accessor + // would query the database again. + const auto snapshot = track.snapshot(); std::cout << track.id() << ". " - << track.title().value_or("(untitled)") << " - " - << track.artist().value_or("(unknown artist)"); + << snapshot.title.value_or("(untitled)") << " - " + << snapshot.artist.value_or("(unknown artist)"); - if (const auto bpm = track.bpm()) - std::cout << " [" << *bpm << " BPM]"; + if (snapshot.bpm) + std::cout << " [" << *snapshot.bpm << " BPM]"; - if (const auto key = track.key()) - std::cout << " [" << *key << "]"; + if (snapshot.key) + std::cout << " [" << *snapshot.key << "]"; - std::cout << "\n " << track.relative_path() << "\n"; + std::cout << "\n " << snapshot.relative_path.value_or("") + << "\n"; } std::cout << "\nPlaylists\n---------\n"; diff --git a/include/djinterop/onelibrary/onelibrary.hpp b/include/djinterop/onelibrary/onelibrary.hpp index ce5dd41..7bf5a67 100644 --- a/include/djinterop/onelibrary/onelibrary.hpp +++ b/include/djinterop/onelibrary/onelibrary.hpp @@ -30,8 +30,9 @@ /// OneLibrary succeeds the DeviceSQL `export.pdb` library that rekordbox wrote /// to USB media; a device usually carries both, and a player that understands /// OneLibrary prefers it. The library is one SQLite database encrypted with -/// SQLCipher 4, whose page format is implemented in `util/crypto`, so neither -/// SQLCipher nor OpenSSL is needed to read one. +/// SQLCipher 4, so reading one needs libdjinterop built with +/// `EXPERIMENTAL_ENABLE_SQLCIPHER`; without it, loading throws +/// `djinterop::unsupported_database`. /// /// Support here is currently **read-only**. Everything that changes a /// database throws `djinterop::unsupported_operation`. @@ -74,9 +75,6 @@ database DJINTEROP_PUBLIC load_database( const std::string& path, const std::string& passphrase = default_passphrase); -/// State shared by everything belonging to one loaded database. -struct onelibrary_context; - } // namespace djinterop::onelibrary #endif // DJINTEROP_ONELIBRARY_ONELIBRARY_HPP diff --git a/include/djinterop/onelibrary/v1/library.hpp b/include/djinterop/onelibrary/v1/library.hpp index ceff539..3e0631b 100644 --- a/include/djinterop/onelibrary/v1/library.hpp +++ b/include/djinterop/onelibrary/v1/library.hpp @@ -36,9 +36,9 @@ namespace djinterop::onelibrary::v1 /// The tables expose the format as the device holds it, translating no /// further than resolving a lookup reference to the text behind it. /// -/// Loading decrypts the device into memory, and derives a key to do it, which -/// is deliberately expensive; take `database()` from a library rather than -/// also calling `load_database`. +/// Loading derives the key that decrypts the device, which is deliberately +/// expensive; take `database()` from a library rather than also calling +/// `load_database`. class DJINTEROP_PUBLIC library { public: @@ -47,17 +47,14 @@ class DJINTEROP_PUBLIC library const std::string& path, const std::string& passphrase = default_passphrase); - /// Construct from a context, as `load_database` builds one. - explicit library(std::shared_ptr context); - /// The `content` table. - [[nodiscard]] content_table content() const { return content_; } + [[nodiscard]] content_table content() const; /// The `playlist` table, and the membership beside it. - [[nodiscard]] playlist_table playlist() const { return playlist_; } + [[nodiscard]] playlist_table playlist() const; /// The `property` table, which records the schema version. - [[nodiscard]] property_table property() const { return property_; } + [[nodiscard]] property_table property() const; /// The device, through the format-agnostic interface. [[nodiscard]] djinterop::database database() const; @@ -68,9 +65,6 @@ class DJINTEROP_PUBLIC library private: std::shared_ptr context_; - content_table content_; - playlist_table playlist_; - property_table property_; }; } // namespace djinterop::onelibrary::v1 diff --git a/include/djinterop/onelibrary/v1/playlist_table.hpp b/include/djinterop/onelibrary/v1/playlist_table.hpp index b0e64e1..e56a8b1 100644 --- a/include/djinterop/onelibrary/v1/playlist_table.hpp +++ b/include/djinterop/onelibrary/v1/playlist_table.hpp @@ -33,18 +33,13 @@ struct onelibrary_context; namespace v1 { -/// Special value for id to indicate that a given row is not a row of the -/// database, and the value a root playlist carries as its parent. -constexpr int64_t PLAYLIST_ROW_ID_NONE = 0; - /// One row of the `playlist` table. struct playlist_row { - int64_t id = PLAYLIST_ROW_ID_NONE; + int64_t id = 0; std::string name; - /// The playlist this one sits under. A root has either no parent - /// recorded or a parent of `PLAYLIST_ROW_ID_NONE`; rekordbox writes both. + /// The playlist this one sits under, or no value for a root. std::optional parent_id; /// Position among siblings, counting from one. diff --git a/src/djinterop/onelibrary/onelibrary.cpp b/src/djinterop/onelibrary/onelibrary.cpp index 7b7ca99..6e78231 100644 --- a/src/djinterop/onelibrary/onelibrary.cpp +++ b/src/djinterop/onelibrary/onelibrary.cpp @@ -17,14 +17,16 @@ #include +#include +#include #include +#include #include +#include #include -#include "../util/crypto/encrypted_database.hpp" -#include "../util/filesystem.hpp" -#include "loader.hpp" +#include "../util/sqlcipher.hpp" #include "onelibrary_context.hpp" #include "v1/database_impl.hpp" @@ -43,73 +45,56 @@ struct resolved_location /// Work out where the database is, given a device or the file itself. resolved_location resolve(const std::string& path) { + if (std::filesystem::is_directory(path)) + return resolved_location{path, path + "/" + database_relative_path}; + // A path that names the database directly implies its device root, which - // is three levels up: `/PIONEER/rekordbox/exportLibrary.db`. - if (!util::path_is_directory(path)) - { - // Walking up by index, rather than by assigning a piece of a string - // back to itself three times over, which is a shape GCC's -Wrestrict - // cannot see the safety of. - auto end = path.size(); - for (int level = 0; level < 3; ++level) - { - const auto separator = end == 0 ? std::string::npos - : path.find_last_of("/\\", end - 1); - - // A relative path with nothing above it sits in the working - // directory, which is then the root of the device. - if (separator == std::string::npos) - return resolved_location{".", path}; - - end = separator; - } - - return resolved_location{path.substr(0, end), path}; - } - - return resolved_location{path, path + "/" + database_relative_path}; + // is three levels up: `/PIONEER/rekordbox/exportLibrary.db`. A + // relative path with nothing above it sits in the working directory, which + // is then the root of the device. + auto root = + std::filesystem::path{path}.parent_path().parent_path().parent_path(); + if (root.empty()) + root = "."; + + return resolved_location{root.string(), path}; } } // anonymous namespace +void verify_schema(onelibrary_context& context) +{ + // A real export has twenty-two tables; demanding the ones this library + // does not read would reject a database that is merely older or newer. + constexpr std::array required_tables{ + "content", "artist", "album", "genre", "label", + "playlist", "playlist_content", "property"}; + + std::set present; + context.db << "SELECT name FROM sqlite_master WHERE type = 'table'" >> + [&](std::string name) { present.insert(std::move(name)); }; + + for (const auto& table : required_tables) + if (present.count(table) == 0) + throw database_inconsistency{ + std::string{"The table `"} + table + + "` is missing, so this is not a OneLibrary database"}; +} + std::shared_ptr load_context( const std::string& path, const std::string& passphrase) { const auto location = resolve(path); - if (!util::path_exists(location.database_path)) + if (!std::filesystem::exists(location.database_path)) throw database_not_found{location.database_path}; - // Key derivation is expensive, so the passphrase is not tested - // separately: a wrong one shows up as the database failing to open. - std::shared_ptr context; - try - { - context = std::make_shared( - location.directory, util::crypto::open_encrypted_database( - location.database_path, passphrase)); - - // Opening a database reads nothing, so touch it here: a wrong - // passphrase would otherwise not be noticed until the first query. - context->db << "SELECT COUNT(*) FROM sqlite_master" >> [](int64_t) {}; - } - catch (const sqlite::sqlite_exception&) - { - throw unsupported_database{ - "The file `" + location.database_path + - "` is not a SQLCipher database that the given passphrase opens"}; - } - catch (const util::crypto::encrypted_database_error& e) - { - throw unsupported_database{e.what()}; - } - catch (const util::crypto::encryption_unsupported& e) - { - throw unsupported_database{e.what()}; - } + auto context = std::make_shared( + location.directory, util::open_encrypted_database( + location.database_path, passphrase)); // Fail here, while the caller still has the path in hand. - v1::database_impl{context}.verify(); + verify_schema(*context); return context; } @@ -117,7 +102,7 @@ std::shared_ptr load_context( bool database_exists(const std::string& path) { const auto location = resolve(path); - return util::path_exists(location.database_path); + return std::filesystem::exists(location.database_path); } database load_database(const std::string& path, const std::string& passphrase) diff --git a/src/djinterop/onelibrary/onelibrary_context.hpp b/src/djinterop/onelibrary/onelibrary_context.hpp index 944d634..6b2b609 100644 --- a/src/djinterop/onelibrary/onelibrary_context.hpp +++ b/src/djinterop/onelibrary/onelibrary_context.hpp @@ -17,6 +17,7 @@ #pragma once +#include #include #include @@ -40,10 +41,21 @@ struct onelibrary_context /// the database file itself sits in. const std::string directory; - /// The database, decrypted from the device into memory. + /// The database, opened with the key that decrypts it. sqlite::database db; }; +/// Open the database on a device and check that it is one. +/// +/// \param path Either the root directory of a device, or the database file. +std::shared_ptr load_context( + const std::string& path, const std::string& passphrase); + +/// Check that a database holds the tables of a OneLibrary one. +/// +/// \throws database_inconsistency If a table is missing. +void verify_schema(onelibrary_context& context); + /// Refuse an operation that would change the database. [[noreturn]] inline void read_only() { diff --git a/src/djinterop/onelibrary/v1/content_table.cpp b/src/djinterop/onelibrary/v1/content_table.cpp index 240bcbc..6ceaf4f 100644 --- a/src/djinterop/onelibrary/v1/content_table.cpp +++ b/src/djinterop/onelibrary/v1/content_table.cpp @@ -22,6 +22,7 @@ #include "../../util/sqlite_query.hpp" #include "../onelibrary_context.hpp" +#include "track_conversion.hpp" namespace djinterop::onelibrary::v1 { @@ -162,13 +163,10 @@ std::vector content_table::all_ids() const std::vector content_table::ids_by_path(const std::string& path) const { - const auto qualified = - !path.empty() && path.front() == '/' ? path : "/" + path; - return util::collect_ids( context_->db, "SELECT content_id FROM content WHERE path = ? ORDER BY content_id", - qualified); + to_device_path(path)); } bool content_table::exists(int64_t id) const diff --git a/src/djinterop/onelibrary/v1/crate_impl.cpp b/src/djinterop/onelibrary/v1/crate_impl.cpp index e8f5b55..04dab41 100644 --- a/src/djinterop/onelibrary/v1/crate_impl.cpp +++ b/src/djinterop/onelibrary/v1/crate_impl.cpp @@ -96,8 +96,7 @@ std::vector crate_impl::tracks() { std::vector results; for (const auto& track_id : playlist_table{context_}.track_ids(id())) - results.push_back( - track{std::make_shared(context_, track_id)}); + results.push_back(make_track(context_, track_id)); return results; } diff --git a/src/djinterop/onelibrary/v1/database_impl.cpp b/src/djinterop/onelibrary/v1/database_impl.cpp index 4d27b2d..0012315 100644 --- a/src/djinterop/onelibrary/v1/database_impl.cpp +++ b/src/djinterop/onelibrary/v1/database_impl.cpp @@ -17,8 +17,6 @@ #include "database_impl.hpp" -#include -#include #include #include @@ -32,18 +30,6 @@ #include "track_impl.hpp" namespace djinterop::onelibrary::v1 { -namespace -{ -/// The tables that must be present for a database to be a OneLibrary one. -/// -/// A real export has twenty-two; demanding the ones this library does not read -/// would reject a database that is merely older or newer. -constexpr std::array required_tables{ - "content", "artist", "album", "genre", "label", - "playlist", "playlist_content", "property"}; - -} // anonymous namespace - database_impl::database_impl(std::shared_ptr context) : djinterop::database_impl{ {feature::supports_nested_crates, feature::supports_nested_playlists, @@ -84,15 +70,7 @@ std::string database_impl::version_name() void database_impl::verify() { - std::set present; - context_->db << "SELECT name FROM sqlite_master WHERE type = 'table'" >> - [&](std::string name) { present.insert(std::move(name)); }; - - for (const auto& table : required_tables) - if (present.count(table) == 0) - throw database_inconsistency{ - std::string{"The table `"} + table + - "` is missing, so this is not a OneLibrary database"}; + verify_schema(*context_); } std::optional database_impl::track_by_id(int64_t id) @@ -100,14 +78,14 @@ std::optional database_impl::track_by_id(int64_t id) if (!content_table{context_}.exists(id)) return std::nullopt; - return track{std::make_shared(context_, id)}; + return make_track(context_, id); } std::vector database_impl::tracks() { std::vector results; for (const auto& id : content_table{context_}.all_ids()) - results.push_back(track{std::make_shared(context_, id)}); + results.push_back(make_track(context_, id)); return results; } @@ -117,7 +95,7 @@ std::vector database_impl::tracks_by_relative_path( { std::vector results; for (const auto& id : content_table{context_}.ids_by_path(relative_path)) - results.push_back(track{std::make_shared(context_, id)}); + results.push_back(make_track(context_, id)); return results; } diff --git a/src/djinterop/onelibrary/v1/library.cpp b/src/djinterop/onelibrary/v1/library.cpp index b53f7f7..6e0902f 100644 --- a/src/djinterop/onelibrary/v1/library.cpp +++ b/src/djinterop/onelibrary/v1/library.cpp @@ -17,22 +17,28 @@ #include -#include - -#include "../loader.hpp" #include "../onelibrary_context.hpp" #include "database_impl.hpp" namespace djinterop::onelibrary::v1 { library::library(const std::string& path, const std::string& passphrase) : - library{load_context(path, passphrase)} + context_{load_context(path, passphrase)} +{ +} + +content_table library::content() const +{ + return content_table{context_}; +} + +playlist_table library::playlist() const { + return playlist_table{context_}; } -library::library(std::shared_ptr context) : - context_{std::move(context)}, content_{context_}, playlist_{context_}, - property_{context_} +property_table library::property() const { + return property_table{context_}; } djinterop::database library::database() const diff --git a/src/djinterop/onelibrary/v1/playlist_impl.cpp b/src/djinterop/onelibrary/v1/playlist_impl.cpp index e17410e..c1cf4d7 100644 --- a/src/djinterop/onelibrary/v1/playlist_impl.cpp +++ b/src/djinterop/onelibrary/v1/playlist_impl.cpp @@ -83,8 +83,7 @@ std::vector playlist_impl::tracks() const { std::vector results; for (const auto& track_id : playlist_table{context_}.track_ids(id_)) - results.push_back( - track{std::make_shared(context_, track_id)}); + results.push_back(make_track(context_, track_id)); return results; } diff --git a/src/djinterop/onelibrary/v1/playlist_impl.hpp b/src/djinterop/onelibrary/v1/playlist_impl.hpp index 8eeccd1..d8fba5c 100644 --- a/src/djinterop/onelibrary/v1/playlist_impl.hpp +++ b/src/djinterop/onelibrary/v1/playlist_impl.hpp @@ -37,7 +37,6 @@ class playlist_impl : public djinterop::playlist_impl public: playlist_impl(std::shared_ptr context, int64_t id); - [[nodiscard]] int64_t id() const noexcept { return id_; } void add_track_back(const djinterop::track_impl& tr) override; void add_track_after( diff --git a/src/djinterop/onelibrary/v1/track_conversion.cpp b/src/djinterop/onelibrary/v1/track_conversion.cpp index ae2715d..543cf88 100644 --- a/src/djinterop/onelibrary/v1/track_conversion.cpp +++ b/src/djinterop/onelibrary/v1/track_conversion.cpp @@ -18,8 +18,8 @@ #include "track_conversion.hpp" #include -#include -#include +#include +#include namespace djinterop::onelibrary::v1 { namespace @@ -65,18 +65,16 @@ std::optional parse_musical_key(const std::string& name) // Notation is a note letter, an optional accidental, and an optional `m` // for a minor key: `C`, `F#m`, `Bb`. Both the ASCII and the typographic // accidentals are accepted. - size_t position = 0; - if (position >= name.size()) + if (name.empty()) return std::nullopt; const auto note = semitones_above_c( - static_cast( - std::toupper(static_cast(name[position])))); + static_cast(std::toupper(static_cast(name[0])))); if (!note) return std::nullopt; auto semitone = *note; - ++position; + size_t position = 1; // Step over an accidental if one is next. The ASCII and typographic // spellings mean the same thing and differ only in how many bytes they @@ -110,68 +108,101 @@ std::optional parse_musical_key(const std::string& name) return std::nullopt; } -track_snapshot to_snapshot(const content_row& row) +std::optional to_bpm(std::optional bpm_x100) { - track_snapshot snapshot; + if (!bpm_x100 || *bpm_x100 <= 0) + return std::nullopt; - snapshot.title = row.title; - snapshot.artist = row.artist; - snapshot.composer = row.composer; - snapshot.album = row.album; - snapshot.genre = row.genre; - snapshot.publisher = row.label; - snapshot.comment = row.comment; + return static_cast(*bpm_x100) / 100; +} - if (row.bpm_x100.has_value() && *row.bpm_x100 > 0) - snapshot.bpm = static_cast(*row.bpm_x100) / 100; +std::optional to_duration( + std::optional length) +{ + if (!length || length->count() <= 0) + return std::nullopt; - if (row.length.has_value() && row.length->count() > 0) - snapshot.duration = - std::chrono::duration_cast(*row.length); + return std::chrono::duration_cast(*length); +} + +std::optional to_positive_int(std::optional value) +{ + if (!value || *value <= 0) + return std::nullopt; - if (row.track_number.has_value() && *row.track_number > 0) - snapshot.track_number = static_cast(*row.track_number); + return static_cast(*value); +} - if (row.release_year.has_value() && *row.release_year > 0) - snapshot.year = static_cast(*row.release_year); +std::optional to_rating(std::optional stars) +{ + if (!stars) + return std::nullopt; // djinterop rates a track from zero to one hundred, where rekordbox uses // whole stars. - if (row.rating_stars.has_value()) - snapshot.rating = - static_cast(std::clamp(*row.rating_stars, 0, 5) * 20); + return static_cast(std::clamp(*stars, 0, 5) * 20); +} - if (row.path.has_value()) - { - // Paths are absolute within the device, whereas djinterop wants them - // relative to the directory of the database. - const auto& path = *row.path; - snapshot.relative_path = path.front() == '/' ? path.substr(1) : path; - } +std::string to_relative_path(const std::string& path) +{ + return !path.empty() && path.front() == '/' ? path.substr(1) : path; +} - if (row.file_size.has_value() && *row.file_size > 0) - snapshot.file_bytes = static_cast(*row.file_size); +std::string to_device_path(const std::string& relative_path) +{ + return "/" + to_relative_path(relative_path); +} - if (row.bitrate.has_value() && *row.bitrate > 0) - snapshot.bitrate = static_cast(*row.bitrate); +std::optional to_sample_rate(std::optional sampling_rate) +{ + if (!sampling_rate || *sampling_rate <= 0) + return std::nullopt; - if (row.sampling_rate.has_value() && *row.sampling_rate > 0) - { - snapshot.sample_rate = static_cast(*row.sampling_rate); + return static_cast(*sampling_rate); +} - // The database records a duration in whole seconds and no sample - // count, so the count can only be recovered to that precision. - if (row.length.has_value() && row.length->count() > 0) - snapshot.sample_count = static_cast( - row.length->count() * *row.sampling_rate); - } +std::optional to_sample_count( + std::optional length, + std::optional sampling_rate) +{ + // The database records a duration in whole seconds and no sample count, + // so the count can only be recovered to that precision. + if (!length || length->count() <= 0 || !sampling_rate || + *sampling_rate <= 0) + return std::nullopt; + + return static_cast(length->count() * *sampling_rate); +} + +track_snapshot to_snapshot(const content_row& row) +{ + track_snapshot snapshot; + + snapshot.title = row.title; + snapshot.artist = row.artist; + snapshot.composer = row.composer; + snapshot.album = row.album; + snapshot.genre = row.genre; + snapshot.publisher = row.label; + snapshot.comment = row.comment; + snapshot.bpm = to_bpm(row.bpm_x100); + snapshot.duration = to_duration(row.length); + snapshot.track_number = to_positive_int(row.track_number); + snapshot.year = to_positive_int(row.release_year); + snapshot.rating = to_rating(row.rating_stars); + snapshot.bitrate = to_positive_int(row.bitrate); + snapshot.sample_rate = to_sample_rate(row.sampling_rate); + snapshot.sample_count = to_sample_count(row.length, row.sampling_rate); + + if (row.path) + snapshot.relative_path = to_relative_path(*row.path); + + if (row.file_size && *row.file_size > 0) + snapshot.file_bytes = static_cast(*row.file_size); - if (row.key.has_value()) + if (row.key) snapshot.key = parse_musical_key(*row.key); - // Beatgrids, waveforms, hot cues and loops are not in the database: - // rekordbox leaves them in the ANLZ files that `analysisDataFilePath` - // points at, and exports an empty `cue` table. return snapshot; } diff --git a/src/djinterop/onelibrary/v1/track_conversion.hpp b/src/djinterop/onelibrary/v1/track_conversion.hpp index 198b52b..a5c690d 100644 --- a/src/djinterop/onelibrary/v1/track_conversion.hpp +++ b/src/djinterop/onelibrary/v1/track_conversion.hpp @@ -17,8 +17,9 @@ #pragma once +#include +#include #include - #include #include @@ -26,6 +27,34 @@ #include namespace djinterop::onelibrary::v1 { +// Each column is interpreted in one place, shared by `to_snapshot` and the +// single-field accessors of `track_impl`, so that the two cannot disagree. +// rekordbox writes zero for a numeric field it does not know. + +[[nodiscard]] std::optional to_bpm(std::optional bpm_x100); + +[[nodiscard]] std::optional to_duration( + std::optional length); + +/// Interpret a count, such as a track number, a year or a bitrate. +[[nodiscard]] std::optional to_positive_int(std::optional value); + +[[nodiscard]] std::optional to_rating(std::optional stars); + +/// Paths are absolute within the device, as `/Contents/...`, whereas djinterop +/// wants them relative to its root. +[[nodiscard]] std::string to_relative_path(const std::string& path); + +/// The inverse of `to_relative_path`. +[[nodiscard]] std::string to_device_path(const std::string& relative_path); + +[[nodiscard]] std::optional to_sample_rate( + std::optional sampling_rate); + +[[nodiscard]] std::optional to_sample_count( + std::optional length, + std::optional sampling_rate); + /// Build a track snapshot from a content row. [[nodiscard]] track_snapshot to_snapshot(const content_row& row); diff --git a/src/djinterop/onelibrary/v1/track_impl.cpp b/src/djinterop/onelibrary/v1/track_impl.cpp index 6c80810..cb010cc8 100644 --- a/src/djinterop/onelibrary/v1/track_impl.cpp +++ b/src/djinterop/onelibrary/v1/track_impl.cpp @@ -17,9 +17,6 @@ #include "track_impl.hpp" -#include -#include -#include #include #include @@ -45,6 +42,11 @@ track_impl::track_impl( { } +track make_track(std::shared_ptr context, int64_t id) +{ + return track{std::make_shared(std::move(context), id)}; +} + track_snapshot track_impl::snapshot() const { const auto row = content_table{context_}.get(id()); @@ -79,12 +81,7 @@ std::vector track_impl::containing_crates() std::string track_impl::relative_path() { const auto path = content_table{context_}.get_path(id()); - if (!path) - return {}; - - // Paths are absolute within the device, whereas djinterop wants them - // relative to the directory of the database. - return path->front() == '/' ? path->substr(1) : *path; + return path ? to_relative_path(*path) : std::string{}; } std::string track_impl::filename() @@ -119,20 +116,12 @@ std::vector track_impl::beatgrid() std::optional track_impl::bitrate() { - const auto bitrate = content_table{context_}.get_bitrate(id()); - if (!bitrate || *bitrate <= 0) - return std::nullopt; - - return static_cast(*bitrate); + return to_positive_int(content_table{context_}.get_bitrate(id())); } std::optional track_impl::bpm() { - const auto bpm_x100 = content_table{context_}.get_bpm_x100(id()); - if (!bpm_x100 || *bpm_x100 <= 0) - return std::nullopt; - - return static_cast(*bpm_x100) / 100; + return to_bpm(content_table{context_}.get_bpm_x100(id())); } std::optional track_impl::comment() @@ -147,11 +136,7 @@ std::optional track_impl::composer() std::optional track_impl::duration() { - const auto length = content_table{context_}.get_length(id()); - if (!length || length->count() <= 0) - return std::nullopt; - - return std::chrono::duration_cast(*length); + return to_duration(content_table{context_}.get_length(id())); } std::optional track_impl::genre() @@ -207,35 +192,19 @@ std::optional track_impl::publisher() std::optional track_impl::rating() { - const auto stars = content_table{context_}.get_rating_stars(id()); - if (!stars) - return std::nullopt; - - // djinterop rates a track from zero to one hundred, where rekordbox uses - // whole stars. - return static_cast(std::clamp(*stars, 0, 5) * 20); + return to_rating(content_table{context_}.get_rating_stars(id())); } std::optional track_impl::sample_count() { - // The database records a duration in whole seconds and no sample count, - // so the count can only be recovered to that precision. const content_table content{context_}; - const auto length = content.get_length(id()); - const auto rate = content.get_sampling_rate(id()); - if (!length || length->count() <= 0 || !rate || *rate <= 0) - return std::nullopt; - - return static_cast(length->count() * *rate); + return to_sample_count( + content.get_length(id()), content.get_sampling_rate(id())); } std::optional track_impl::sample_rate() { - const auto rate = content_table{context_}.get_sampling_rate(id()); - if (!rate || *rate <= 0) - return std::nullopt; - - return static_cast(*rate); + return to_sample_rate(content_table{context_}.get_sampling_rate(id())); } std::optional track_impl::title() @@ -245,11 +214,7 @@ std::optional track_impl::title() std::optional track_impl::track_number() { - const auto number = content_table{context_}.get_track_number(id()); - if (!number || *number <= 0) - return std::nullopt; - - return static_cast(*number); + return to_positive_int(content_table{context_}.get_track_number(id())); } std::vector track_impl::waveform() @@ -259,11 +224,7 @@ std::vector track_impl::waveform() std::optional track_impl::year() { - const auto year = content_table{context_}.get_release_year(id()); - if (!year || *year <= 0) - return std::nullopt; - - return static_cast(*year); + return to_positive_int(content_table{context_}.get_release_year(id())); } void track_impl::update(const track_snapshot&) diff --git a/src/djinterop/onelibrary/v1/track_impl.hpp b/src/djinterop/onelibrary/v1/track_impl.hpp index 3cd500c..499e744 100644 --- a/src/djinterop/onelibrary/v1/track_impl.hpp +++ b/src/djinterop/onelibrary/v1/track_impl.hpp @@ -112,4 +112,7 @@ class track_impl : public djinterop::track_impl std::shared_ptr context_; }; +/// Wrap a content row as a track. +track make_track(std::shared_ptr context, int64_t id); + } // namespace djinterop::onelibrary::v1 diff --git a/src/djinterop/util/crypto/aes.cpp b/src/djinterop/util/crypto/aes.cpp deleted file mode 100644 index 6406e32..0000000 --- a/src/djinterop/util/crypto/aes.cpp +++ /dev/null @@ -1,353 +0,0 @@ -/* - This file is part of libdjinterop. - - libdjinterop is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libdjinterop is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with libdjinterop. If not, see . - */ - -#include "aes.hpp" - -#include -#include - -#include "aes_hardware.hpp" - -namespace djinterop::util::crypto -{ -namespace -{ -constexpr int key_words = 8; - -inline uint8_t rotl8(uint8_t x, unsigned shift) noexcept -{ - return static_cast((x << shift) | (x >> (8 - shift))); -} - -/// Multiply by x in GF(2^8) modulo the AES polynomial. -inline uint8_t xtime(uint8_t x) noexcept -{ - return static_cast((x << 1) ^ ((x & 0x80) ? 0x1b : 0x00)); -} - -/// Multiply two elements of GF(2^8) modulo the AES polynomial. -constexpr uint8_t gmul(uint8_t a, uint8_t b) noexcept -{ - uint8_t result = 0; - while (b != 0) - { - if (b & 1) - result ^= a; - a = static_cast((a << 1) ^ ((a & 0x80) ? 0x1b : 0x00)); - b = static_cast(b >> 1); - } - return result; -} - -inline uint32_t load_be32(const uint8_t* p) noexcept -{ - return (static_cast(p[0]) << 24) | - (static_cast(p[1]) << 16) | - (static_cast(p[2]) << 8) | static_cast(p[3]); -} - -inline void store_be32(uint8_t* p, uint32_t v) noexcept -{ - p[0] = static_cast(v >> 24); - p[1] = static_cast(v >> 16); - p[2] = static_cast(v >> 8); - p[3] = static_cast(v); -} - -/// Pack four field elements into a column, most significant byte first. -constexpr uint32_t column(uint8_t r0, uint8_t r1, uint8_t r2, uint8_t r3) -{ - return (static_cast(r0) << 24) | - (static_cast(r1) << 16) | - (static_cast(r2) << 8) | static_cast(r3); -} - -/// The substitution boxes, and the round tables built on top of them. -/// -/// The boxes are derived rather than tabulated: each entry is the affine -/// transform of the multiplicative inverse in GF(2^8), which the standard walk -/// over p = 3^i, q = 3^-i enumerates in a single pass. -/// -/// The round tables fold substitution and column mixing together, so a round -/// costs four lookups and four exclusive-ors per column rather than the field -/// arithmetic the definition calls for. Only the first table of each set is -/// held: the other three are rotations of it, and a rotation is cheaper than a -/// second cache line. -struct aes_tables -{ - uint8_t forward[256]; - uint8_t inverse[256]; - uint32_t encrypt[256]; - uint32_t decrypt[256]; - - aes_tables() noexcept : forward{}, inverse{}, encrypt{}, decrypt{} - { - uint8_t p = 1; - uint8_t q = 1; - do - { - p = static_cast(p ^ (p << 1) ^ ((p & 0x80) ? 0x1b : 0)); - - // q = q / 3, i.e. q multiplied by the inverse of 3. - q ^= static_cast(q << 1); - q ^= static_cast(q << 2); - q ^= static_cast(q << 4); - if (q & 0x80) - q ^= 0x09; - - const auto value = static_cast( - q ^ rotl8(q, 1) ^ rotl8(q, 2) ^ rotl8(q, 3) ^ rotl8(q, 4) ^ - 0x63); - forward[p] = value; - inverse[value] = p; - } while (p != 1); - - // Zero has no multiplicative inverse; it maps to the affine constant. - forward[0] = 0x63; - inverse[0x63] = 0x00; - - for (int i = 0; i < 256; ++i) - { - // The column MixColumns makes of a byte standing alone in row 0, - // and the one InvMixColumns makes of the same. - const auto s = forward[i]; - encrypt[i] = column(gmul(s, 2), s, s, gmul(s, 3)); - - const auto t = inverse[i]; - decrypt[i] = - column(gmul(t, 14), gmul(t, 9), gmul(t, 13), gmul(t, 11)); - } - } -}; - -const aes_tables& tables() noexcept -{ - static const aes_tables instance; - return instance; -} - -/// Extract the byte of a column that ShiftRows will place in a given row. -inline uint8_t row_of(uint32_t c, int row) noexcept -{ - return static_cast(c >> (24 - (8 * row))); -} - -/// Apply InvMixColumns to a round key in place, turning the forward schedule -/// into the one the equivalent inverse cipher wants. -void inverse_mix_columns(uint8_t* key) noexcept -{ - for (int i = 0; i < 4; ++i) - { - auto* c = key + (4 * i); - const uint8_t a0 = c[0], a1 = c[1], a2 = c[2], a3 = c[3]; - c[0] = static_cast( - gmul(a0, 14) ^ gmul(a1, 11) ^ gmul(a2, 13) ^ gmul(a3, 9)); - c[1] = static_cast( - gmul(a0, 9) ^ gmul(a1, 14) ^ gmul(a2, 11) ^ gmul(a3, 13)); - c[2] = static_cast( - gmul(a0, 13) ^ gmul(a1, 9) ^ gmul(a2, 14) ^ gmul(a3, 11)); - c[3] = static_cast( - gmul(a0, 11) ^ gmul(a1, 13) ^ gmul(a2, 9) ^ gmul(a3, 14)); - } -} - -/// The number of 32-bit words in a key schedule. -constexpr int schedule_words = 4 * (aes256_rounds + 1); - -/// Which way ShiftRows moves the rows along. -constexpr int forwards = 1; -constexpr int backwards = -1; - -/// Unpack a schedule into words once, rather than reassembling four bytes at -/// every use: a page is thousands of blocks, and a schedule is sixty words. -void unpack_schedule(const uint8_t* keys, uint32_t* words) noexcept -{ - for (int i = 0; i < schedule_words; ++i) - words[i] = load_be32(keys + (4 * i)); -} - -/// One block through the round tables, in either direction. -/// -/// Once decryption goes through the equivalent inverse cipher the two -/// directions have the same shape, and differ only in which tables they read -/// and which way the rows shift. -template -void transform_block( - const uint32_t* keys, const uint32_t* table, const uint8_t* box, - const uint8_t* input, uint8_t* output) noexcept -{ - // ShiftRows draws row r of an output column from the column r steps away, - // forwards when encrypting and backwards when decrypting. - const auto from = [](int c, int row) - { return (c + (direction * row) + 4) & 3; }; - - uint32_t s[4]; - for (int i = 0; i < 4; ++i) - s[i] = load_be32(input + (4 * i)) ^ keys[i]; - - for (int round = 1; round < aes256_rounds; ++round) - { - const auto* rk = keys + (4 * round); - - uint32_t u[4]; - for (int c = 0; c < 4; ++c) - { - u[c] = table[row_of(s[from(c, 0)], 0)] ^ - std::rotr(table[row_of(s[from(c, 1)], 1)], 8) ^ - std::rotr(table[row_of(s[from(c, 2)], 2)], 16) ^ - std::rotr(table[row_of(s[from(c, 3)], 3)], 24) ^ rk[c]; - } - - std::memcpy(s, u, sizeof(s)); - } - - // The last round leaves out the column mixing, so it reads the box direct. - const auto* rk = keys + (4 * aes256_rounds); - for (int c = 0; c < 4; ++c) - { - const auto substituted = column( - box[row_of(s[from(c, 0)], 0)], box[row_of(s[from(c, 1)], 1)], - box[row_of(s[from(c, 2)], 2)], box[row_of(s[from(c, 3)], 3)]); - - store_be32(output + (4 * c), substituted ^ rk[c]); - } -} - -} // anonymous namespace - -aes256_cbc::aes256_cbc( - const uint8_t* key, aes_implementation implementation) noexcept : - round_keys_{}, inverse_round_keys_{}, - hardware_{ - implementation == aes_implementation::automatic && - hardware::aes_available()} -{ - const auto& sbox = tables().forward; - - std::memcpy(round_keys_.data(), key, aes256_key_length); - - uint8_t rcon = 1; - for (int word = key_words; word < 4 * (aes256_rounds + 1); ++word) - { - uint8_t temp[4]; - std::memcpy(temp, round_keys_.data() + (4 * (word - 1)), 4); - - if (word % key_words == 0) - { - // Rotate, substitute, and add the round constant. - const auto first = temp[0]; - temp[0] = static_cast(sbox[temp[1]] ^ rcon); - temp[1] = sbox[temp[2]]; - temp[2] = sbox[temp[3]]; - temp[3] = sbox[first]; - rcon = xtime(rcon); - } - else if (word % key_words == 4) - { - for (auto& byte : temp) - byte = sbox[byte]; - } - - for (int i = 0; i < 4; ++i) - { - round_keys_[(4 * word) + i] = - round_keys_[(4 * (word - key_words)) + i] ^ temp[i]; - } - } - - // Reverse the schedule, and fold InvMixColumns into every round key but - // the two on the ends, which are only ever added and never mixed. - for (int round = 0; round <= aes256_rounds; ++round) - { - auto* destination = - inverse_round_keys_.data() + (aes_block_length * round); - std::memcpy( - destination, - round_keys_.data() + (aes_block_length * (aes256_rounds - round)), - aes_block_length); - - if (round != 0 && round != aes256_rounds) - inverse_mix_columns(destination); - } -} - -void aes256_cbc::encrypt( - const uint8_t* iv, const uint8_t* input, uint8_t* output, - size_t length) const noexcept -{ - if (hardware_) - { - hardware::aes256_cbc_encrypt( - round_keys_.data(), iv, input, output, length); - return; - } - - const auto& t = tables(); - - uint32_t schedule[schedule_words]; - unpack_schedule(round_keys_.data(), schedule); - - uint8_t chain[aes_block_length]; - std::memcpy(chain, iv, aes_block_length); - - for (size_t offset = 0; offset < length; offset += aes_block_length) - { - uint8_t block[aes_block_length]; - for (size_t i = 0; i < aes_block_length; ++i) - block[i] = input[offset + i] ^ chain[i]; - - transform_block( - schedule, t.encrypt, t.forward, block, output + offset); - std::memcpy(chain, output + offset, aes_block_length); - } -} - -void aes256_cbc::decrypt( - const uint8_t* iv, const uint8_t* input, uint8_t* output, - size_t length) const noexcept -{ - if (hardware_) - { - hardware::aes256_cbc_decrypt( - inverse_round_keys_.data(), iv, input, output, length); - return; - } - - const auto& t = tables(); - - uint32_t schedule[schedule_words]; - unpack_schedule(inverse_round_keys_.data(), schedule); - - uint8_t chain[aes_block_length]; - std::memcpy(chain, iv, aes_block_length); - - for (size_t offset = 0; offset < length; offset += aes_block_length) - { - // Retain the ciphertext before writing, in case output aliases input. - uint8_t next_chain[aes_block_length]; - std::memcpy(next_chain, input + offset, aes_block_length); - - transform_block( - schedule, t.decrypt, t.inverse, input + offset, output + offset); - for (size_t i = 0; i < aes_block_length; ++i) - output[offset + i] ^= chain[i]; - - std::memcpy(chain, next_chain, aes_block_length); - } -} - -} // namespace djinterop::util::crypto diff --git a/src/djinterop/util/crypto/aes.hpp b/src/djinterop/util/crypto/aes.hpp deleted file mode 100644 index 954825b..0000000 --- a/src/djinterop/util/crypto/aes.hpp +++ /dev/null @@ -1,98 +0,0 @@ -/* - This file is part of libdjinterop. - - libdjinterop is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libdjinterop is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with libdjinterop. If not, see . - */ - -#pragma once - -#include -#include -#include - -namespace djinterop::util::crypto -{ -constexpr size_t aes_block_length = 16; -constexpr size_t aes256_key_length = 32; - -/// Number of rounds in AES-256, and hence one less than the number of round -/// keys that its schedule expands to. -constexpr int aes256_rounds = 14; - -constexpr size_t aes256_schedule_length = - aes_block_length * (aes256_rounds + 1); - -/// Which implementation an instance should use. -enum class aes_implementation -{ - /// The processor's AES instructions where it has them, tables otherwise. - automatic, - - /// The tables, whatever the processor offers. Nothing in the library asks - /// for this; it lets tests reach the fallback on a machine that would - /// otherwise never run it. - tabulated, -}; - -/// AES-256 in cipher block chaining mode, as specified by FIPS 197 and -/// NIST SP 800-38A. -/// -/// No padding scheme is applied: input lengths must be a whole number of -/// blocks. SQLCipher pages are always block-aligned by construction, which is -/// why a padding mode is not needed. -/// -/// Where the processor has AES instructions they are used, and tables -/// otherwise. Neither resists timing analysis, and the tables plainly do not; -/// the passphrase of a OneLibrary database is a constant compiled into -/// rekordbox rather than a secret, so there is nothing to learn from it. -class aes256_cbc -{ -public: - /// Construct a cipher for a given key, which must be - /// `aes256_key_length` bytes long. - explicit aes256_cbc( - const uint8_t* key, aes_implementation implementation = - aes_implementation::automatic) noexcept; - - /// Encrypt `length` bytes from `input` into `output`, which may alias - /// `input`. `length` must be a multiple of the AES block length. - /// The initialisation vector must be `aes_block_length` bytes long. - void encrypt( - const uint8_t* iv, const uint8_t* input, uint8_t* output, - size_t length) const noexcept; - - /// Decrypt `length` bytes from `input` into `output`, which may alias - /// `input`. `length` must be a multiple of the AES block length. - /// The initialisation vector must be `aes_block_length` bytes long. - void decrypt( - const uint8_t* iv, const uint8_t* input, uint8_t* output, - size_t length) const noexcept; - -private: - /// Expanded key schedule: 15 round keys of 16 bytes each. - alignas(16) std::array round_keys_; - - /// The schedule of the equivalent inverse cipher: the round keys reversed, - /// all but the outermost two passed through InvMixColumns. - /// - /// Folding that transform into the keys lets the inverse rounds take the - /// same shape as the forward ones, which is what both the tables and the - /// processor's AES instructions expect. - alignas(16) std::array inverse_round_keys_; - - /// Whether the processor this program is running on has AES instructions. - bool hardware_; -}; - -} // namespace djinterop::util::crypto diff --git a/src/djinterop/util/crypto/aes_hardware.cpp b/src/djinterop/util/crypto/aes_hardware.cpp deleted file mode 100644 index c254f5d..0000000 --- a/src/djinterop/util/crypto/aes_hardware.cpp +++ /dev/null @@ -1,321 +0,0 @@ -/* - This file is part of libdjinterop. - - libdjinterop is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libdjinterop is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with libdjinterop. If not, see . - */ - -#include "aes_hardware.hpp" - -#include "aes.hpp" - -// One of these is defined by the build system, if it found a way to compile the -// intrinsics for the target. Anything else falls back to the tables in -// `aes.cpp`. -#if defined(DJINTEROP_AES_INTRINSICS_X86) -#if defined(_MSC_VER) -#include -#else -#include -#endif -#elif defined(DJINTEROP_AES_INTRINSICS_ARM64) -#include -#if defined(_WIN32) -#include -#elif defined(__linux__) -#include -#if !defined(HWCAP_AES) -#define HWCAP_AES (1 << 3) -#endif -#elif defined(__FreeBSD__) -#include -#include -#endif -#endif - -#if defined(DJINTEROP_AES_INTRINSICS_X86) || \ - defined(DJINTEROP_AES_INTRINSICS_ARM64) -#define DJINTEROP_AES_INTRINSICS 1 -#endif - -namespace djinterop::util::crypto::hardware -{ -#if defined(DJINTEROP_AES_INTRINSICS) - -namespace -{ -// Each architecture supplies the same handful of operations, over which the -// chaining below is written once. Rounds are counted as x86 counts them: -// AArch64 adds its round key at the start of a round rather than the end, so -// its idiom looks a round out of step while doing the same work. - -#if defined(DJINTEROP_AES_INTRINSICS_X86) - -using block = __m128i; - -inline block load(const uint8_t* p) noexcept -{ - return _mm_loadu_si128(reinterpret_cast(p)); -} - -inline void store(uint8_t* p, block x) noexcept -{ - _mm_storeu_si128(reinterpret_cast<__m128i*>(p), x); -} - -inline block block_xor(block a, block b) noexcept -{ - return _mm_xor_si128(a, b); -} - -inline block encrypt_first(block x, const block* rk) noexcept -{ - return _mm_xor_si128(x, rk[0]); -} - -inline block encrypt_round(block x, const block* rk, int round) noexcept -{ - return _mm_aesenc_si128(x, rk[round + 1]); -} - -inline block encrypt_last(block x, const block* rk) noexcept -{ - return _mm_aesenclast_si128(x, rk[aes256_rounds]); -} - -inline block decrypt_first(block x, const block* dk) noexcept -{ - return _mm_xor_si128(x, dk[0]); -} - -inline block decrypt_round(block x, const block* dk, int round) noexcept -{ - return _mm_aesdec_si128(x, dk[round + 1]); -} - -inline block decrypt_last(block x, const block* dk) noexcept -{ - return _mm_aesdeclast_si128(x, dk[aes256_rounds]); -} - -#else - -using block = uint8x16_t; - -inline block load(const uint8_t* p) noexcept -{ - return vld1q_u8(p); -} - -inline void store(uint8_t* p, block x) noexcept -{ - vst1q_u8(p, x); -} - -inline block block_xor(block a, block b) noexcept -{ - return veorq_u8(a, b); -} - -inline block encrypt_first(block x, const block*) noexcept -{ - return x; -} - -inline block encrypt_round(block x, const block* rk, int round) noexcept -{ - return vaesmcq_u8(vaeseq_u8(x, rk[round])); -} - -inline block encrypt_last(block x, const block* rk) noexcept -{ - return veorq_u8(vaeseq_u8(x, rk[aes256_rounds - 1]), rk[aes256_rounds]); -} - -inline block decrypt_first(block x, const block*) noexcept -{ - return x; -} - -inline block decrypt_round(block x, const block* dk, int round) noexcept -{ - return vaesimcq_u8(vaesdq_u8(x, dk[round])); -} - -inline block decrypt_last(block x, const block* dk) noexcept -{ - return veorq_u8(vaesdq_u8(x, dk[aes256_rounds - 1]), dk[aes256_rounds]); -} - -#endif - -/// Rounds between the first and the last, of which both directions have the -/// same number. -constexpr int middle_rounds = aes256_rounds - 1; - -/// Blocks decrypted together. -/// -/// The AES instructions take several cycles to yield a result but accept a new -/// block every cycle, so independent blocks run several times faster than a -/// dependent chain does. Eight covers the latency without exhausting the -/// sixteen vector registers both architectures have. -constexpr size_t lanes = 8; - -void load_schedule(const uint8_t* keys, block* out) noexcept -{ - for (int i = 0; i <= aes256_rounds; ++i) - out[i] = load(keys + (aes_block_length * i)); -} - -inline block decrypt_block(block x, const block* dk) noexcept -{ - x = decrypt_first(x, dk); - for (int round = 0; round < middle_rounds; ++round) - x = decrypt_round(x, dk, round); - - return decrypt_last(x, dk); -} - -} // anonymous namespace - -void aes256_cbc_encrypt( - const uint8_t* round_keys, const uint8_t* iv, const uint8_t* input, - uint8_t* output, size_t length) noexcept -{ - block rk[aes256_rounds + 1]; - load_schedule(round_keys, rk); - - auto chain = load(iv); - for (size_t offset = 0; offset < length; offset += aes_block_length) - { - auto x = encrypt_first(block_xor(load(input + offset), chain), rk); - for (int round = 0; round < middle_rounds; ++round) - x = encrypt_round(x, rk, round); - - chain = encrypt_last(x, rk); - store(output + offset, chain); - } -} - -void aes256_cbc_decrypt( - const uint8_t* inverse_round_keys, const uint8_t* iv, const uint8_t* input, - uint8_t* output, size_t length) noexcept -{ - block dk[aes256_rounds + 1]; - load_schedule(inverse_round_keys, dk); - - auto chain = load(iv); - size_t offset = 0; - - constexpr size_t stride = lanes * aes_block_length; - for (; offset + stride <= length; offset += stride) - { - // Every block of the group is read before any is written, so the - // ciphertext each one chains with survives an output that aliases the - // input. - block ciphertext[lanes]; - block x[lanes]; - for (size_t lane = 0; lane < lanes; ++lane) - { - ciphertext[lane] = load(input + offset + (lane * aes_block_length)); - x[lane] = decrypt_first(ciphertext[lane], dk); - } - - for (int round = 0; round < middle_rounds; ++round) - for (auto& lane : x) - lane = decrypt_round(lane, dk, round); - - for (auto& lane : x) - lane = decrypt_last(lane, dk); - - x[0] = block_xor(x[0], chain); - for (size_t lane = 1; lane < lanes; ++lane) - x[lane] = block_xor(x[lane], ciphertext[lane - 1]); - - chain = ciphertext[lanes - 1]; - for (size_t lane = 0; lane < lanes; ++lane) - store(output + offset + (lane * aes_block_length), x[lane]); - } - - for (; offset < length; offset += aes_block_length) - { - const auto ciphertext = load(input + offset); - store(output + offset, block_xor(decrypt_block(ciphertext, dk), chain)); - chain = ciphertext; - } -} - -bool aes_available() noexcept -{ -#if defined(DJINTEROP_AES_INTRINSICS_X86) && defined(_MSC_VER) - // AES-NI is bit 25 of ECX for CPUID leaf 1. - static const bool available = [] - { - int registers[4] = {0, 0, 0, 0}; - __cpuid(registers, 1); - return (registers[2] & (1 << 25)) != 0; - }(); - return available; -#elif defined(DJINTEROP_AES_INTRINSICS_X86) - static const bool available = __builtin_cpu_supports("aes"); - return available; -#elif defined(__APPLE__) - // Every processor Apple has shipped in an arm64 device implements the - // cryptographic extension, and the platform guarantees it. - return true; -#elif defined(_WIN32) - static const bool available = - IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != - FALSE; - return available; -#elif defined(__linux__) - static const bool available = (getauxval(AT_HWCAP) & HWCAP_AES) != 0; - return available; -#elif defined(__FreeBSD__) - static const bool available = [] - { - unsigned long capabilities = 0; - if (elf_aux_info(AT_HWCAP, &capabilities, sizeof(capabilities)) != 0) - return false; - - return (capabilities & HWCAP_AES) != 0; - }(); - return available; -#else - // With no way to ask, assume not: the fallback is correct, only slower. - return false; -#endif -} - -#else - -bool aes_available() noexcept -{ - return false; -} - -// Never reached: `aes_available` says so, and every caller asks first. - -void aes256_cbc_encrypt( - const uint8_t*, const uint8_t*, const uint8_t*, uint8_t*, size_t) noexcept -{ -} - -void aes256_cbc_decrypt( - const uint8_t*, const uint8_t*, const uint8_t*, uint8_t*, size_t) noexcept -{ -} - -#endif - -} // namespace djinterop::util::crypto::hardware diff --git a/src/djinterop/util/crypto/aes_hardware.hpp b/src/djinterop/util/crypto/aes_hardware.hpp deleted file mode 100644 index 3a4fe28..0000000 --- a/src/djinterop/util/crypto/aes_hardware.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - This file is part of libdjinterop. - - libdjinterop is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libdjinterop is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with libdjinterop. If not, see . - */ - -#pragma once - -#include -#include - -/// The processor's own AES instructions, where it has them. -/// -/// x86 has had AES-NI since 2010 and AArch64 the ARMv8 cryptographic extension -/// from the start, so in practice these are what runs, and the tables in -/// `aes.cpp` are the fallback. Whether the instructions are there is a -/// property of the processor rather than of the build, so it is asked at run -/// time and this is the only translation unit compiled with them enabled. -namespace djinterop::util::crypto::hardware -{ -/// Whether the processor running this program has AES instructions. The -/// underlying query is made once and remembered. -[[nodiscard]] bool aes_available() noexcept; - -/// Encrypt in cipher block chaining mode. `length` is a whole number of -/// blocks, `output` may alias `input`, and `round_keys` is the forward -/// schedule of 15 round keys. -void aes256_cbc_encrypt( - const uint8_t* round_keys, const uint8_t* iv, const uint8_t* input, - uint8_t* output, size_t length) noexcept; - -/// Decrypt in cipher block chaining mode, taking the schedule of the -/// equivalent inverse cipher, which is the form both instruction sets expect. -/// -/// Chaining constrains encryption to one block at a time, but not decryption: -/// a block needs only its own ciphertext and the one before it, so several go -/// at once here to keep the pipeline of the AES instructions full. -void aes256_cbc_decrypt( - const uint8_t* inverse_round_keys, const uint8_t* iv, const uint8_t* input, - uint8_t* output, size_t length) noexcept; - -} // namespace djinterop::util::crypto::hardware diff --git a/src/djinterop/util/crypto/encrypted_database.hpp b/src/djinterop/util/crypto/encrypted_database.hpp deleted file mode 100644 index 2c58a58..0000000 --- a/src/djinterop/util/crypto/encrypted_database.hpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - This file is part of libdjinterop. - - libdjinterop is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libdjinterop is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with libdjinterop. If not, see . - */ - -#pragma once - -#include -#include - -#include - -namespace djinterop::util::crypto -{ -// Exactly one implementation of this header is compiled in. The one beside -// it decrypts the pages itself; a SQLCipher-backed one would open the file -// directly and replace that source file. - -/// Thrown when a file is not an encrypted database the passphrase opens. -class encrypted_database_error : public std::runtime_error -{ -public: - explicit encrypted_database_error(const std::string& what) : - std::runtime_error{what} - { - } -}; - -/// Thrown when this build cannot open encrypted databases at all. -class encryption_unsupported : public std::runtime_error -{ -public: - explicit encryption_unsupported(const std::string& what) : - std::runtime_error{what} - { - } -}; - -/// Test whether this build can open encrypted databases. -[[nodiscard]] bool encrypted_databases_supported() noexcept; - -/// Open an encrypted database for reading. -/// -/// The connection serves the decrypted contents, with any write-ahead log -/// already folded in. It is not a handle on the file. -/// -/// \throws encryption_unsupported If this build cannot open encrypted -/// databases. -/// \throws encrypted_database_error If the passphrase does not open the file. -[[nodiscard]] sqlite::database open_encrypted_database( - const std::string& path, const std::string& passphrase); - -} // namespace djinterop::util::crypto diff --git a/src/djinterop/util/crypto/encrypted_database_builtin.cpp b/src/djinterop/util/crypto/encrypted_database_builtin.cpp deleted file mode 100644 index 93e43ca..0000000 --- a/src/djinterop/util/crypto/encrypted_database_builtin.cpp +++ /dev/null @@ -1,107 +0,0 @@ -/* - This file is part of libdjinterop. - - libdjinterop is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libdjinterop is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with libdjinterop. If not, see . - */ - -// The implementation of `encrypted_database.hpp` that decrypts SQLCipher -// pages itself, rather than by linking a copy of SQLCipher. - -#include "encrypted_database.hpp" - -#include -#include -#include -#include - -#include - -#include "sqlcipher_codec.hpp" -#include "sqlcipher_wal.hpp" - -// The decrypted image is handed to SQLite through `sqlite3_deserialize`, from -// SQLite 3.36, which can also be compiled out. -#if defined(SQLITE_OMIT_DESERIALIZE) || SQLITE_VERSION_NUMBER < 3036000 -#define DJINTEROP_HAVE_DESERIALIZE 0 -#else -#define DJINTEROP_HAVE_DESERIALIZE 1 -#endif - -namespace djinterop::util::crypto -{ -bool encrypted_databases_supported() noexcept -{ - return DJINTEROP_HAVE_DESERIALIZE; -} - -sqlite::database open_encrypted_database( - const std::string& path, const std::string& passphrase) -{ -#if !DJINTEROP_HAVE_DESERIALIZE - (void)passphrase; - throw encryption_unsupported{ - "This build of libdjinterop cannot read the encrypted database `" + - path + - "`. Reading one goes through `sqlite3_deserialize`, which needs " - "SQLite 3.36 or newer, built without SQLITE_OMIT_DESERIALIZE, and " - "this build was made against an older or narrower SQLite."}; -#else - // Key derivation is expensive, so it is done once here. - const auto codec = make_codec_for(path, passphrase); - if (!codec) - throw encrypted_database_error{ - "The file `" + path + "` is too small to be a database"}; - - std::vector image; - try - { - // A device is checkpointed on eject but still declares itself - // write-ahead-logged, which SQLite will not open read-only without - // the log. Fold it in and rewrite the header first. - image = decrypt_database_to_image(path, *codec); - } - catch (const sqlcipher_error&) - { - throw encrypted_database_error{ - "The file `" + path + - "` is not a SQLCipher database that the given passphrase opens"}; - } - - sqlite::database db{":memory:"}; - - // SQLite frees the buffer with the connection, so it must come from - // SQLite's own allocator. - auto* buffer = static_cast(sqlite3_malloc64(image.size())); - if (buffer == nullptr) - throw std::bad_alloc{}; - - std::memcpy(buffer, image.data(), image.size()); - - const auto rc = sqlite3_deserialize( - db.connection().get(), "main", buffer, - static_cast(image.size()), - static_cast(image.size()), - SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_READONLY); - if (rc != SQLITE_OK) - throw encrypted_database_error{ - "The database `" + path + - "` could not be read once it had been decrypted"}; - - return db; -#endif -} - -} // namespace djinterop::util::crypto - -#undef DJINTEROP_HAVE_DESERIALIZE diff --git a/src/djinterop/util/crypto/sha512.cpp b/src/djinterop/util/crypto/sha512.cpp deleted file mode 100644 index 85ab43f..0000000 --- a/src/djinterop/util/crypto/sha512.cpp +++ /dev/null @@ -1,336 +0,0 @@ -/* - This file is part of libdjinterop. - - libdjinterop is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libdjinterop is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with libdjinterop. If not, see . - */ - -#include "sha512.hpp" - -#include -#include -#include - -namespace djinterop::util::crypto -{ -namespace -{ -constexpr uint64_t k[80] = { - 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fULL, - 0xe9b5dba58189dbbcULL, 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, - 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, 0xd807aa98a3030242ULL, - 0x12835b0145706fbeULL, 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, - 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, 0x9bdc06a725c71235ULL, - 0xc19bf174cf692694ULL, 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, - 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, 0x2de92c6f592b0275ULL, - 0x4a7484aa6ea6e483ULL, 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, - 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, 0xb00327c898fb213fULL, - 0xbf597fc7beef0ee4ULL, 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, - 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, 0x27b70a8546d22ffcULL, - 0x2e1b21385c26c926ULL, 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, - 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, 0x81c2c92e47edaee6ULL, - 0x92722c851482353bULL, 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, - 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, 0xd192e819d6ef5218ULL, - 0xd69906245565a910ULL, 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, - 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, 0x2748774cdf8eeb99ULL, - 0x34b0bcb5e19b48a8ULL, 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, - 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, 0x748f82ee5defb2fcULL, - 0x78a5636f43172f60ULL, 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, - 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 0xbef9a3f7b2c67915ULL, - 0xc67178f2e372532bULL, 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, - 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, 0x06f067aa72176fbaULL, - 0x0a637dc5a2c898a6ULL, 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, - 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, 0x3c9ebe0a15c9bebcULL, - 0x431d67c49c100d4cULL, 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, - 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL}; - -inline uint64_t rotr(uint64_t x, unsigned n) noexcept -{ - return (x >> n) | (x << (64 - n)); -} - -/// Extend the message schedule in place. Only the last sixteen words are ever -/// needed, so they live in a window that wraps rather than in the array of -/// eighty the specification describes. -inline uint64_t extend(uint64_t* w, int j) noexcept -{ - const auto x = w[(j + 1) & 15]; - const auto y = w[(j + 14) & 15]; - w[j] += (rotr(x, 1) ^ rotr(x, 8) ^ (x >> 7)) + w[(j + 9) & 15] + - (rotr(y, 19) ^ rotr(y, 61) ^ (y >> 6)); - return w[j]; -} - -/// One round, over working variables named in the order the round expects them. -/// -/// The specification shifts the eight variables along by one each round; the -/// caller below rotates their *names* instead, which is free. Sixteen rounds -/// are two whole turns of the eight, so a block ends with every name back -/// where it started. -#define DJINTEROP_SHA512_ROUND(a, b, c, d, e, f, g, h, word, constant) \ - do \ - { \ - const uint64_t t1 = \ - (h) + (rotr((e), 14) ^ rotr((e), 18) ^ rotr((e), 41)) + \ - (((e) & (f)) ^ (~(e) & (g))) + (constant) + (word); \ - const uint64_t t2 = (rotr((a), 28) ^ rotr((a), 34) ^ rotr((a), 39)) + \ - (((a) & (b)) ^ ((a) & (c)) ^ ((b) & (c))); \ - (d) += t1; \ - (h) = t1 + t2; \ - } while (false) - -#define DJINTEROP_SHA512_BLOCK(word) \ - do \ - { \ - DJINTEROP_SHA512_ROUND(a, b, c, d, e, f, g, h, word(0), k[i + 0]); \ - DJINTEROP_SHA512_ROUND(h, a, b, c, d, e, f, g, word(1), k[i + 1]); \ - DJINTEROP_SHA512_ROUND(g, h, a, b, c, d, e, f, word(2), k[i + 2]); \ - DJINTEROP_SHA512_ROUND(f, g, h, a, b, c, d, e, word(3), k[i + 3]); \ - DJINTEROP_SHA512_ROUND(e, f, g, h, a, b, c, d, word(4), k[i + 4]); \ - DJINTEROP_SHA512_ROUND(d, e, f, g, h, a, b, c, word(5), k[i + 5]); \ - DJINTEROP_SHA512_ROUND(c, d, e, f, g, h, a, b, word(6), k[i + 6]); \ - DJINTEROP_SHA512_ROUND(b, c, d, e, f, g, h, a, word(7), k[i + 7]); \ - DJINTEROP_SHA512_ROUND(a, b, c, d, e, f, g, h, word(8), k[i + 8]); \ - DJINTEROP_SHA512_ROUND(h, a, b, c, d, e, f, g, word(9), k[i + 9]); \ - DJINTEROP_SHA512_ROUND(g, h, a, b, c, d, e, f, word(10), k[i + 10]); \ - DJINTEROP_SHA512_ROUND(f, g, h, a, b, c, d, e, word(11), k[i + 11]); \ - DJINTEROP_SHA512_ROUND(e, f, g, h, a, b, c, d, word(12), k[i + 12]); \ - DJINTEROP_SHA512_ROUND(d, e, f, g, h, a, b, c, word(13), k[i + 13]); \ - DJINTEROP_SHA512_ROUND(c, d, e, f, g, h, a, b, word(14), k[i + 14]); \ - DJINTEROP_SHA512_ROUND(b, c, d, e, f, g, h, a, word(15), k[i + 15]); \ - i += 16; \ - } while (false) - -/// The first sixteen rounds read the block as it arrives; the rest extend the -/// schedule a word at a time, just before the round that consumes it. -#define DJINTEROP_SHA512_LOADED(j) w[j] -#define DJINTEROP_SHA512_EXTENDED(j) extend(w, j) - -inline uint64_t load_be64(const uint8_t* p) noexcept -{ - uint64_t v = 0; - for (int i = 0; i < 8; ++i) - v = (v << 8) | p[i]; - return v; -} - -inline void store_be64(uint8_t* p, uint64_t v) noexcept -{ - for (int i = 7; i >= 0; --i) - { - p[i] = static_cast(v & 0xff); - v >>= 8; - } -} - -} // anonymous namespace - -sha512::sha512() noexcept : - state_{0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, 0x3c6ef372fe94f82bULL, - 0xa54ff53a5f1d36f1ULL, 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, - 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL}, - buffer_{}, buffered_{0}, total_length_{0} -{ -} - -void sha512::compress(const uint8_t* block) noexcept -{ - uint64_t w[16]; - for (int j = 0; j < 16; ++j) - w[j] = load_be64(block + (j * 8)); - - auto a = state_[0], b = state_[1], c = state_[2], d = state_[3]; - auto e = state_[4], f = state_[5], g = state_[6], h = state_[7]; - - int i = 0; - DJINTEROP_SHA512_BLOCK(DJINTEROP_SHA512_LOADED); - DJINTEROP_SHA512_BLOCK(DJINTEROP_SHA512_EXTENDED); - DJINTEROP_SHA512_BLOCK(DJINTEROP_SHA512_EXTENDED); - DJINTEROP_SHA512_BLOCK(DJINTEROP_SHA512_EXTENDED); - DJINTEROP_SHA512_BLOCK(DJINTEROP_SHA512_EXTENDED); - - state_[0] += a; - state_[1] += b; - state_[2] += c; - state_[3] += d; - state_[4] += e; - state_[5] += f; - state_[6] += g; - state_[7] += h; -} - -void sha512::update(const uint8_t* data, size_t length) noexcept -{ - total_length_ += length; - - if (buffered_ > 0) - { - const auto take = std::min(length, sha512_block_length - buffered_); - std::memcpy(buffer_.data() + buffered_, data, take); - buffered_ += take; - data += take; - length -= take; - - if (buffered_ < sha512_block_length) - return; - - compress(buffer_.data()); - buffered_ = 0; - } - - while (length >= sha512_block_length) - { - compress(data); - data += sha512_block_length; - length -= sha512_block_length; - } - - std::memcpy(buffer_.data(), data, length); - buffered_ = length; -} - -sha512_digest sha512::finalise() noexcept -{ - // The length field is 128 bits wide, but inputs here are far below 2^64 - // bytes, so the high half is always zero. - const uint64_t bit_length = total_length_ * 8; - - buffer_[buffered_++] = 0x80; - if (buffered_ > sha512_block_length - 16) - { - std::fill(buffer_.begin() + buffered_, buffer_.end(), uint8_t{0}); - compress(buffer_.data()); - buffered_ = 0; - } - - std::fill(buffer_.begin() + buffered_, buffer_.end() - 8, uint8_t{0}); - store_be64(buffer_.data() + sha512_block_length - 8, bit_length); - compress(buffer_.data()); - - sha512_digest digest{}; - for (int i = 0; i < 8; ++i) - store_be64(digest.data() + (i * 8), state_[i]); - return digest; -} - -sha512_digest sha512::hash(const uint8_t* data, size_t length) noexcept -{ - sha512 h; - h.update(data, length); - return h.finalise(); -} - -hmac_sha512_key::hmac_sha512_key(const uint8_t* key, size_t key_length) noexcept -{ - std::array padded{}; - if (key_length > sha512_block_length) - { - const auto digest = sha512::hash(key, key_length); - std::copy(digest.begin(), digest.end(), padded.begin()); - } - else - { - std::copy(key, key + key_length, padded.begin()); - } - - std::array pad{}; - for (size_t i = 0; i < sha512_block_length; ++i) - pad[i] = static_cast(padded[i] ^ 0x36); - inner_.update(pad.data(), pad.size()); - - for (size_t i = 0; i < sha512_block_length; ++i) - pad[i] = static_cast(padded[i] ^ 0x5c); - outer_.update(pad.data(), pad.size()); -} - -hmac_sha512_stream::hmac_sha512_stream(const hmac_sha512_key& key) noexcept : - inner_{key.inner_}, outer_{key.outer_} -{ -} - -void hmac_sha512_stream::update(const uint8_t* data, size_t length) noexcept -{ - inner_.update(data, length); -} - -sha512_digest hmac_sha512_stream::finalise() noexcept -{ - const auto inner_digest = inner_.finalise(); - outer_.update(inner_digest.data(), inner_digest.size()); - return outer_.finalise(); -} - -sha512_digest hmac_sha512( - const uint8_t* key, size_t key_length, const uint8_t* data, - size_t data_length) noexcept -{ - const hmac_sha512_key prepared{key, key_length}; - - hmac_sha512_stream stream{prepared}; - stream.update(data, data_length); - return stream.finalise(); -} - -std::vector pbkdf2_hmac_sha512( - const uint8_t* password, size_t password_length, const uint8_t* salt, - size_t salt_length, uint32_t iterations, size_t length) -{ - if (iterations == 0) - throw std::invalid_argument{"PBKDF2 requires at least one iteration"}; - - // Every iteration below authenticates under the same key, so its padded - // blocks are absorbed once here rather than a quarter of a million times. - const hmac_sha512_key prf{password, password_length}; - - std::vector output; - output.reserve(length); - - for (uint32_t index = 1; output.size() < length; ++index) - { - // U_1 = PRF(password, salt || INT_BE32(index)) - const uint8_t counter[4] = { - static_cast(index >> 24), - static_cast(index >> 16), static_cast(index >> 8), - static_cast(index)}; - - hmac_sha512_stream first{prf}; - first.update(salt, salt_length); - first.update(counter, sizeof(counter)); - - auto u = first.finalise(); - auto accumulator = u; - - for (uint32_t i = 1; i < iterations; ++i) - { - hmac_sha512_stream next{prf}; - next.update(u.data(), u.size()); - u = next.finalise(); - - for (size_t j = 0; j < accumulator.size(); ++j) - accumulator[j] ^= u[j]; - } - - const auto take = std::min(accumulator.size(), length - output.size()); - output.insert( - output.end(), accumulator.begin(), accumulator.begin() + take); - } - - return output; -} - -} // namespace djinterop::util::crypto - -#undef DJINTEROP_SHA512_ROUND -#undef DJINTEROP_SHA512_BLOCK -#undef DJINTEROP_SHA512_LOADED -#undef DJINTEROP_SHA512_EXTENDED diff --git a/src/djinterop/util/crypto/sha512.hpp b/src/djinterop/util/crypto/sha512.hpp deleted file mode 100644 index 4f4380a..0000000 --- a/src/djinterop/util/crypto/sha512.hpp +++ /dev/null @@ -1,99 +0,0 @@ -/* - This file is part of libdjinterop. - - libdjinterop is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libdjinterop is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with libdjinterop. If not, see . - */ - -#pragma once - -#include -#include -#include -#include - -namespace djinterop::util::crypto -{ -constexpr size_t sha512_digest_length = 64; -constexpr size_t sha512_block_length = 128; - -using sha512_digest = std::array; - -/// Streaming SHA-512, as specified by FIPS 180-4. -class sha512 -{ -public: - sha512() noexcept; - - void update(const uint8_t* data, size_t length) noexcept; - - /// Finalise the hash. The object must not be reused afterwards. - [[nodiscard]] sha512_digest finalise() noexcept; - - static sha512_digest hash(const uint8_t* data, size_t length) noexcept; - -private: - void compress(const uint8_t* block) noexcept; - - std::array state_; - std::array buffer_; - size_t buffered_; - uint64_t total_length_; -}; - -/// A key prepared for repeated HMAC-SHA-512. -/// -/// Both padded key blocks are absorbed once, here, so a message afterwards -/// costs only the compressions its own bytes call for. That halves key -/// derivation, and every page of a database is checked under the one key. -class hmac_sha512_key -{ -public: - hmac_sha512_key(const uint8_t* key, size_t key_length) noexcept; - -private: - friend class hmac_sha512_stream; - - sha512 inner_; - sha512 outer_; -}; - -/// One message authenticated under a prepared key. A stream holds no -/// reference to the key, so streams built from one key on several threads do -/// not interfere. -class hmac_sha512_stream -{ -public: - explicit hmac_sha512_stream(const hmac_sha512_key& key) noexcept; - - void update(const uint8_t* data, size_t length) noexcept; - - /// Finalise the tag. The object must not be reused afterwards. - [[nodiscard]] sha512_digest finalise() noexcept; - -private: - sha512 inner_; - sha512 outer_; -}; - -/// Compute HMAC-SHA-512, as specified by RFC 2104. -sha512_digest hmac_sha512( - const uint8_t* key, size_t key_length, const uint8_t* data, - size_t data_length) noexcept; - -/// Derive `length` bytes using PBKDF2-HMAC-SHA-512, as specified by RFC 8018. -std::vector pbkdf2_hmac_sha512( - const uint8_t* password, size_t password_length, const uint8_t* salt, - size_t salt_length, uint32_t iterations, size_t length); - -} // namespace djinterop::util::crypto diff --git a/src/djinterop/util/crypto/sqlcipher_codec.cpp b/src/djinterop/util/crypto/sqlcipher_codec.cpp deleted file mode 100644 index fafa3d2..0000000 --- a/src/djinterop/util/crypto/sqlcipher_codec.cpp +++ /dev/null @@ -1,203 +0,0 @@ -/* - This file is part of libdjinterop. - - libdjinterop is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libdjinterop is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with libdjinterop. If not, see . - */ - -#include "sqlcipher_codec.hpp" - -#include - -#include -#include - -namespace djinterop::util::crypto -{ -namespace -{ -constexpr size_t page_key_length = aes256_key_length; -constexpr size_t hmac_key_length = 32; - -/// Derive the page encryption key from the passphrase and the database salt. -std::vector derive_page_key( - const std::string& passphrase, const sqlcipher_salt& salt, - uint32_t iterations) -{ - return pbkdf2_hmac_sha512( - reinterpret_cast(passphrase.data()), passphrase.size(), - salt.data(), salt.size(), iterations, page_key_length); -} - -/// Derive the HMAC key, and prepare it for the pages checked under it. -/// -/// It comes from the *page key*, not the passphrase, using the database salt -/// with every byte XORed by 0x3a. -hmac_sha512_key make_hmac_key( - const std::vector& page_key, const sqlcipher_salt& salt, - uint32_t iterations) -{ - sqlcipher_salt hmac_salt{}; - std::transform( - salt.begin(), salt.end(), hmac_salt.begin(), - [](uint8_t byte) { return static_cast(byte ^ 0x3a); }); - - const auto key = pbkdf2_hmac_sha512( - page_key.data(), page_key.size(), hmac_salt.data(), hmac_salt.size(), - iterations, hmac_key_length); - - return hmac_sha512_key{key.data(), key.size()}; -} - -/// Reject parameters that no SQLCipher database could have, before any time -/// is spent on key derivation. -const sqlcipher_parameters& validated(const sqlcipher_parameters& params) -{ - if (params.reserve < aes_block_length + sha512_digest_length) - throw sqlcipher_error{ - "SQLCipher reserve is too small to hold an IV and an HMAC tag"}; - - if (params.page_size <= params.reserve) - throw sqlcipher_error{ - "SQLCipher page size does not exceed its reserved area"}; - - if (params.payload_size() % aes_block_length != 0) - throw sqlcipher_error{ - "SQLCipher page payload is not a whole number of AES blocks"}; - - return params; -} - -/// Compare two byte sequences without leaking their contents through timing. -bool equal_in_constant_time( - const uint8_t* left, const uint8_t* right, size_t length) noexcept -{ - uint8_t difference = 0; - for (size_t i = 0; i < length; ++i) - difference |= static_cast(left[i] ^ right[i]); - return difference == 0; -} - -} // anonymous namespace - -sqlcipher_codec::sqlcipher_codec( - const std::string& passphrase, const sqlcipher_salt& salt, - const sqlcipher_parameters& params) : - sqlcipher_codec{ - derive_page_key(passphrase, salt, validated(params).kdf_iterations), - salt, params} -{ -} - -sqlcipher_codec::sqlcipher_codec( - const std::vector& page_key, const sqlcipher_salt& salt, - const sqlcipher_parameters& params) : - params_{params}, cipher_{page_key.data()}, - hmac_key_{make_hmac_key(page_key, salt, params.hmac_kdf_iterations)} -{ -} - -sha512_digest sqlcipher_codec::page_mac( - uint32_t page_number, const uint8_t* ciphertext, size_t length, - const uint8_t* iv) const noexcept -{ - // The tag covers ciphertext || IV || page number, the last a little-endian - // 32-bit integer. The three are fed in as they lie rather than gathered - // into a buffer, which would copy every page of a database for nothing. - const uint8_t number[4] = { - static_cast(page_number), - static_cast(page_number >> 8), - static_cast(page_number >> 16), - static_cast(page_number >> 24)}; - - hmac_sha512_stream stream{hmac_key_}; - stream.update(ciphertext, length); - stream.update(iv, aes_block_length); - stream.update(number, sizeof(number)); - return stream.finalise(); -} - -bool sqlcipher_codec::page_mac_is_valid( - uint32_t page_number, const uint8_t* encrypted) const noexcept -{ - const auto offset = ciphertext_offset(page_number); - const auto length = params_.payload_size() - offset; - const auto* iv = encrypted + params_.payload_size(); - const auto* tag = iv + aes_block_length; - - const auto expected = page_mac(page_number, encrypted + offset, length, iv); - return equal_in_constant_time(expected.data(), tag, sha512_digest_length); -} - -void sqlcipher_codec::decrypt_page( - uint32_t page_number, const uint8_t* encrypted, uint8_t* decrypted) const -{ - const auto offset = ciphertext_offset(page_number); - const auto length = params_.payload_size() - offset; - const auto* iv = encrypted + params_.payload_size(); - - if (!page_mac_is_valid(page_number, encrypted)) - throw sqlcipher_error{ - "page " + std::to_string(page_number) + - " failed its integrity check: wrong passphrase, corrupt data, or " - "not a SQLCipher database"}; - - if (offset > 0 && decrypted != encrypted) - std::memcpy(decrypted, encrypted, offset); - - cipher_.decrypt(iv, encrypted + offset, decrypted + offset, length); - - // Leave the reserved area zeroed. SQLite never looks at it once the - // header declares a reserve size, and zeroing keeps the IV and tag of the - // encrypted page from lingering in the decrypted image. - std::memset(decrypted + params_.payload_size(), 0, params_.reserve); -} - -namespace -{ -/// Read the salt of a SQLCipher database from its first page. -sqlcipher_salt read_salt(const uint8_t* first_page) noexcept -{ - sqlcipher_salt salt{}; - std::copy(first_page, first_page + salt.size(), salt.begin()); - return salt; -} - -/// Read the first page of a database, or nothing if there is not one. -std::optional> read_first_page( - const std::string& database_path, size_t page_size) -{ - std::vector page(page_size); - std::ifstream file{database_path, std::ios::binary}; - if (!file.read( - reinterpret_cast(page.data()), - static_cast(page.size()))) - return std::nullopt; - - return page; -} - -} // anonymous namespace - -std::optional make_codec_for( - const std::string& database_path, const std::string& passphrase, - const sqlcipher_parameters& params) -{ - const auto page = read_first_page(database_path, params.page_size); - if (!page) - return std::nullopt; - - return sqlcipher_codec{passphrase, read_salt(page->data()), params}; -} - -} // namespace djinterop::util::crypto diff --git a/src/djinterop/util/crypto/sqlcipher_codec.hpp b/src/djinterop/util/crypto/sqlcipher_codec.hpp deleted file mode 100644 index 0c2fa5f..0000000 --- a/src/djinterop/util/crypto/sqlcipher_codec.hpp +++ /dev/null @@ -1,161 +0,0 @@ -/* - This file is part of libdjinterop. - - libdjinterop is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libdjinterop is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with libdjinterop. If not, see . - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include "aes.hpp" -#include "sha512.hpp" - -namespace djinterop::util::crypto -{ -/// Parameters of a SQLCipher database. -/// -/// The defaults are those of SQLCipher 4, which is what the AlphaTheta -/// OneLibrary `exportLibrary.db` is written with. -struct sqlcipher_parameters -{ - /// Size of a database page, in bytes, including the reserved area. - size_t page_size = 4096; - - /// Bytes reserved at the end of each page: a 16-byte initialisation - /// vector followed by a 64-byte HMAC-SHA-512 tag. - size_t reserve = 80; - - /// Number of PBKDF2 iterations used to derive the page key. - uint32_t kdf_iterations = 256000; - - /// Number of PBKDF2 iterations used to derive the HMAC key. The input is - /// already a strong key, so stretching it further would cost time for - /// nothing. - uint32_t hmac_kdf_iterations = 2; - - [[nodiscard]] size_t payload_size() const noexcept - { - return page_size - reserve; - } -}; - -constexpr size_t sqlcipher_salt_length = 16; - -using sqlcipher_salt = std::array; - -/// The bytes that open a plain SQLite file, and which SQLCipher overwrites -/// with the key derivation salt. -/// -/// The two occupy the same 16 bytes, so a decrypted first page becomes a -/// loadable one by copying this over its salt. -constexpr const char sqlite_file_magic[] = "SQLite format 3"; -constexpr size_t sqlite_file_magic_length = sqlcipher_salt_length; - -/// Thrown when a page fails to decrypt, or fails its integrity check. -class sqlcipher_error : public std::runtime_error -{ -public: - explicit sqlcipher_error(const std::string& what) : std::runtime_error{what} - { - } -}; - -/// Decrypts the pages of a SQLCipher database. -/// -/// Each page holds `payload_size()` bytes of ciphertext, a random -/// initialisation vector, and an HMAC-SHA-512 tag over the ciphertext, the -/// vector, and the page number -- which binds a page to its position, so pages -/// cannot be swapped without detection. -/// -/// The first page is special: its leading 16 bytes are the key derivation -/// salt, in the clear where a plain SQLite file has its header magic, so its -/// ciphertext is 16 bytes shorter than that of every other page. -class sqlcipher_codec -{ -public: - /// Construct a codec for a passphrase and the salt of a database. - /// - /// Key derivation is deliberately expensive, so construct a codec once per - /// database rather than once per page. - sqlcipher_codec( - const std::string& passphrase, const sqlcipher_salt& salt, - const sqlcipher_parameters& params = {}); - - [[nodiscard]] const sqlcipher_parameters& parameters() const noexcept - { - return params_; - } - - /// Decrypt one page. - /// - /// `encrypted` and `decrypted` are both `page_size` bytes long and may - /// alias each other. Page numbers are one-based, as in SQLite itself, and - /// the reserved area of the output is zeroed. - /// - /// On page one the salt is left in place, so a caller that needs a - /// loadable SQLite image must overwrite those bytes with the header magic. - void decrypt_page( - uint32_t page_number, const uint8_t* encrypted, - uint8_t* decrypted) const; - -private: - /// Test whether a page carries a valid HMAC tag. - /// - /// A wrong passphrase fails here rather than yielding plausible noise. - [[nodiscard]] bool page_mac_is_valid( - uint32_t page_number, const uint8_t* encrypted) const noexcept; - - /// Adopt an already-derived page key. The HMAC key comes from the page - /// key rather than the passphrase, so the public constructor derives once - /// and delegates here. - sqlcipher_codec( - const std::vector& page_key, const sqlcipher_salt& salt, - const sqlcipher_parameters& params); - - /// Offset within a page at which its ciphertext begins. - [[nodiscard]] size_t ciphertext_offset(uint32_t page_number) const noexcept - { - return page_number == 1 ? sqlcipher_salt_length : 0; - } - - [[nodiscard]] sha512_digest page_mac( - uint32_t page_number, const uint8_t* ciphertext, size_t length, - const uint8_t* iv) const noexcept; - - sqlcipher_parameters params_; - aes256_cbc cipher_; - - /// The HMAC key, its padded blocks already absorbed. - hmac_sha512_key hmac_key_; -}; - -/// Build the codec of a database, reading its salt from the file. -/// -/// Returns nothing if the file cannot be read or is shorter than a page. Key -/// derivation is deliberately expensive, so a caller that goes on to read more -/// than one page should build the codec once, here, and keep it. -/// -/// \throws sqlcipher_error If the key cannot be derived. -std::optional make_codec_for( - const std::string& database_path, const std::string& passphrase, - const sqlcipher_parameters& params = {}); - -} // namespace djinterop::util::crypto diff --git a/src/djinterop/util/crypto/sqlcipher_wal.cpp b/src/djinterop/util/crypto/sqlcipher_wal.cpp deleted file mode 100644 index 7034ee0..0000000 --- a/src/djinterop/util/crypto/sqlcipher_wal.cpp +++ /dev/null @@ -1,349 +0,0 @@ -/* - This file is part of libdjinterop. - - libdjinterop is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libdjinterop is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with libdjinterop. If not, see . - */ - -#include "sqlcipher_wal.hpp" - -#include -#include -#include -#include -#include -#include - -namespace djinterop::util::crypto -{ -namespace -{ -constexpr size_t wal_header_length = 32; -constexpr size_t wal_frame_header_length = 24; - -/// The two magic numbers that open a log, differing in the byte order used -/// for its checksums. -constexpr uint32_t wal_magic_little_endian = 0x377f0682; -constexpr uint32_t wal_magic_big_endian = 0x377f0683; - -/// Offsets of the two version bytes in the SQLite header. -constexpr size_t write_version_offset = 18; -constexpr size_t read_version_offset = 19; -constexpr uint8_t rollback_journal_version = 1; - -uint32_t load_be32(const uint8_t* p) noexcept -{ - return (static_cast(p[0]) << 24) | - (static_cast(p[1]) << 16) | - (static_cast(p[2]) << 8) | static_cast(p[3]); -} - -uint32_t load_le32(const uint8_t* p) noexcept -{ - return (static_cast(p[3]) << 24) | - (static_cast(p[2]) << 16) | - (static_cast(p[1]) << 8) | static_cast(p[0]); -} - -/// Read a whole file, or nothing if it cannot be read. -std::vector read_file(const std::string& path) -{ - std::ifstream file{path, std::ios::binary | std::ios::ate}; - if (!file) - return {}; - - const auto size = static_cast(file.tellg()); - if (size <= 0) - return {}; - - std::vector contents(static_cast(size)); - file.seekg(0); - if (!file.read( - reinterpret_cast(contents.data()), - static_cast(size))) - return {}; - - return contents; -} - -/// The running checksum SQLite keeps over the contents of a log. -/// -/// Each step folds a pair of 32-bit words into the pair of accumulators, so -/// the checksum of a frame depends on every frame before it: a log cannot be -/// truncated or reordered without detection. -struct wal_checksum -{ - uint32_t s0 = 0; - uint32_t s1 = 0; - - void accumulate( - const uint8_t* data, size_t length, bool big_endian) noexcept - { - for (size_t offset = 0; offset + 8 <= length; offset += 8) - { - const auto first = big_endian ? load_be32(data + offset) - : load_le32(data + offset); - const auto second = big_endian ? load_be32(data + offset + 4) - : load_le32(data + offset + 4); - s0 += first + s1; - s1 += second + s0; - } - } - - /// Compare against a stored pair of checksums. - /// - /// The byte order of the magic number governs how the *contents* are read - /// while accumulating, but the two stored values are header fields, and - /// every header field in a log is big-endian. - [[nodiscard]] bool matches(const uint8_t* expected) const noexcept - { - return s0 == load_be32(expected) && s1 == load_be32(expected + 4); - } -}; - -/// The pages of a log that a reader should honour. -struct wal_contents -{ - /// Page number and ciphertext offset of every committed frame, in the - /// order the log wrote them. - std::vector> frames; - - /// Size the database is to be truncated or extended to, in pages, or zero - /// if the log commits nothing. - uint32_t page_count = 0; -}; - -/// Walk the frames of a log, keeping those up to the last one that committed: -/// anything after it belongs to a transaction that never finished, exactly as -/// SQLite's own recovery decides. -/// -/// \throws sqlcipher_error If the log is not one, or does not go with the -/// database beside it. -wal_contents read_log( - const std::vector& log, const std::string& database_path, - size_t page_size) -{ - wal_contents contents; - if (log.size() < wal_header_length) - return contents; - - const auto magic = load_be32(log.data()); - const auto big_endian_checksums = magic == wal_magic_big_endian; - if (magic != wal_magic_little_endian && !big_endian_checksums) - throw sqlcipher_error{ - "`" + database_path + - "-wal` does not begin like a write-ahead log"}; - - const auto log_page_size = load_be32(log.data() + 8); - if (log_page_size != page_size) - throw sqlcipher_error{ - "the write-ahead log of `" + database_path + - "` uses a different page size from the database"}; - - const auto salt_1 = load_be32(log.data() + 16); - const auto salt_2 = load_be32(log.data() + 20); - - wal_checksum running; - running.accumulate(log.data(), 24, big_endian_checksums); - if (!running.matches(log.data() + 24)) - throw sqlcipher_error{ - "the write-ahead log of `" + database_path + - "` has a damaged header"}; - - const auto frame_length = wal_frame_header_length + page_size; - size_t committed_frames = 0; - - for (size_t offset = wal_header_length; offset + frame_length <= log.size(); - offset += frame_length) - { - const auto* frame = log.data() + offset; - const auto page_number = load_be32(frame); - const auto truncate_to = load_be32(frame + 4); - - // A frame written after the log was reset carries the salt of the - // previous incarnation, and is not part of it. - if (load_be32(frame + 8) != salt_1 || load_be32(frame + 12) != salt_2) - break; - - auto candidate = running; - candidate.accumulate(frame, 8, big_endian_checksums); - candidate.accumulate( - frame + wal_frame_header_length, page_size, big_endian_checksums); - if (!candidate.matches(frame + 16)) - break; - - running = candidate; - contents.frames.emplace_back( - page_number, offset + wal_frame_header_length); - - if (truncate_to != 0) - { - committed_frames = contents.frames.size(); - contents.page_count = truncate_to; - } - } - - contents.frames.resize(committed_frames); - return contents; -} - -/// The invariants of decrypting an image: everything a page needs but its own -/// number. -struct page_work -{ - const sqlcipher_codec& codec; - - /// Where each page's ciphertext lies, or null for one that is in neither - /// the database file nor the log. - const std::vector& sources; - - uint8_t* image; - size_t page_size; -}; - -/// Decrypt a run of pages, whose sources are already settled. -void decrypt_range(const page_work& work, size_t first, size_t last) -{ - for (size_t index = first; index < last; ++index) - { - const auto* encrypted = work.sources[index]; - if (encrypted == nullptr) - continue; - - work.codec.decrypt_page( - static_cast(index + 1), encrypted, - work.image + (index * work.page_size)); - } -} - -/// Decrypt every page, over as many processors as there is work for. -/// -/// A page carries its own initialisation vector and is bound to its own number, -/// so none depends on any other and the work divides by simple arithmetic. -void decrypt_pages(const page_work& work) -{ - // Enough pages that a thread earns the cost of starting it. - constexpr size_t pages_per_worker = 256; - constexpr size_t worker_limit = 16; - - const auto page_count = work.sources.size(); - auto workers = std::min(page_count / pages_per_worker, worker_limit); - workers = std::min(workers, std::thread::hardware_concurrency()); - - if (workers < 2) - { - decrypt_range(work, 0, page_count); - return; - } - - std::vector failures(workers); - const auto share = (page_count + workers - 1) / workers; - - const auto run = [&](size_t worker) - { - try - { - const auto first = worker * share; - decrypt_range(work, first, std::min(page_count, first + share)); - } - catch (...) - { - failures[worker] = std::current_exception(); - } - }; - - std::vector threads; - threads.reserve(workers - 1); - for (size_t worker = 1; worker < workers; ++worker) - threads.emplace_back(run, worker); - - run(0); - - for (auto& thread : threads) - thread.join(); - - // Report the failure nearest the start of the database, so the error a - // caller sees does not depend on how the work happened to divide. - for (const auto& failure : failures) - { - if (failure) - std::rethrow_exception(failure); - } -} - -} // anonymous namespace - -std::vector decrypt_database_to_image( - const std::string& database_path, const sqlcipher_codec& codec) -{ - const auto page_size = codec.parameters().page_size; - - auto image = read_file(database_path); - if (image.size() < page_size) - throw sqlcipher_error{ - "`" + database_path + "` is too small to be a database"}; - - if (image.size() % page_size != 0) - throw sqlcipher_error{ - "`" + database_path + "` is not a whole number of pages"}; - - const auto database_pages = image.size() / page_size; - - // The log is read first: it settles how large the finished image is, and - // which pages of the database file are worth decrypting at all. - const auto log = read_file(database_path + "-wal"); - const auto contents = read_log(log, database_path, page_size); - - const auto image_pages = contents.page_count != 0 - ? static_cast(contents.page_count) - : database_pages; - - // Decryption is in place, so the image is sized first rather than being - // copied out of a second buffer afterwards. - image.resize(image_pages * page_size, 0); - - // Settle where each page's ciphertext lies before decrypting any of it. A - // page the log replaces is never read from the database file, and rekordbox - // leaves most of a fresh export in the log, so that saves the greater part - // of the work. A page in neither keeps the zeroes the resize gave it. - std::vector sources(image_pages, nullptr); - const auto pages_in_both = std::min(image_pages, database_pages); - for (size_t index = 0; index < pages_in_both; ++index) - sources[index] = image.data() + (index * page_size); - - for (const auto& [page_number, payload_offset] : contents.frames) - { - if (page_number == 0 || page_number > image_pages) - continue; - - sources[page_number - 1] = log.data() + payload_offset; - } - - decrypt_pages({codec, sources, image.data(), page_size}); - - // Page one opens with the salt, where a plain database has its magic. - std::memcpy(image.data(), sqlite_file_magic, sqlite_file_magic_length); - - // The image no longer has a log, so mark it as using a rollback journal. - // Left as it is, SQLite would look for the log that has just been folded - // in, and refuse to open the database read-only without it. - if (image.size() > read_version_offset) - { - image[write_version_offset] = rollback_journal_version; - image[read_version_offset] = rollback_journal_version; - } - - return image; -} - -} // namespace djinterop::util::crypto diff --git a/src/djinterop/util/crypto/sqlcipher_wal.hpp b/src/djinterop/util/crypto/sqlcipher_wal.hpp deleted file mode 100644 index 2843368..0000000 --- a/src/djinterop/util/crypto/sqlcipher_wal.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - This file is part of libdjinterop. - - libdjinterop is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libdjinterop is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with libdjinterop. If not, see . - */ - -#pragma once - -#include -#include -#include -#include - -#include "sqlcipher_codec.hpp" - -namespace djinterop::util::crypto -{ -/// Decrypt a SQLCipher database, merging its write-ahead log, into a plain -/// SQLite image. -/// -/// rekordbox leaves most of a fresh export in the log rather than in the -/// database file, so a reader that ignores it reports a nearly empty library -/// with no error at all. -/// -/// The returned image is a standard, unencrypted SQLite file, marked as using -/// a rollback journal: everything the log held has already been folded in. -/// -/// Only the pages the finished image is made of are decrypted, and so only -/// those are authenticated: a page the log replaces, or one past the size the -/// log truncates the database to, is never read from the database file and a -/// fault in it goes unreported. SQLite would not have looked at it either. -/// -/// \param database_path Path of the encrypted database. -/// \param codec Codec built for the database, as by `make_codec_for`. -/// \return Returns a plain SQLite image. -/// \throws sqlcipher_error If the database cannot be read or decrypted. -std::vector decrypt_database_to_image( - const std::string& database_path, const sqlcipher_codec& codec); - -} // namespace djinterop::util::crypto diff --git a/src/djinterop/util/filesystem.cpp b/src/djinterop/util/filesystem.cpp index b6a3e7d..03a61d3 100644 --- a/src/djinterop/util/filesystem.cpp +++ b/src/djinterop/util/filesystem.cpp @@ -44,15 +44,6 @@ bool path_exists(const std::string& directory) return (stat(directory.c_str(), &buf) == 0); } -bool path_is_directory(const std::string& path) -{ - struct stat buf; - if (stat(path.c_str(), &buf) != 0) - return false; - - return (buf.st_mode & S_IFMT) == S_IFDIR; -} - std::string get_filename(const std::string& file_path) { // TODO (haslersn): How to handle Windows path separator? diff --git a/src/djinterop/util/filesystem.hpp b/src/djinterop/util/filesystem.hpp index 4e09216..5c72f8c 100644 --- a/src/djinterop/util/filesystem.hpp +++ b/src/djinterop/util/filesystem.hpp @@ -24,7 +24,6 @@ namespace djinterop::util { void create_dir(const std::string& directory); bool path_exists(const std::string& directory); -bool path_is_directory(const std::string& path); std::string get_filename(const std::string& file_path); std::optional get_file_extension(const std::string& file_path); diff --git a/src/djinterop/util/sqlcipher.cpp b/src/djinterop/util/sqlcipher.cpp new file mode 100644 index 0000000..21be2c8 --- /dev/null +++ b/src/djinterop/util/sqlcipher.cpp @@ -0,0 +1,58 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#include "sqlcipher.hpp" + +#include + +#include + +#include + +namespace djinterop::util +{ +sqlite::database open_encrypted_database( + const std::string& path, const std::string& passphrase) +{ + // A write-ahead-logged database cannot be read without the shared-memory + // index beside it, so even a read-only connection creates one if it is + // missing. + sqlite3* raw = nullptr; + const auto open_rc = + sqlite3_open_v2(path.c_str(), &raw, SQLITE_OPEN_READONLY, nullptr); + std::shared_ptr connection{raw, sqlite3_close_v2}; + if (open_rc != SQLITE_OK) + throw unsupported_database{ + "The database `" + path + "` could not be opened"}; + + // Setting the key reads nothing, so read something: a wrong passphrase + // would otherwise not be noticed until the first query. + const std::unique_ptr sql{ + sqlite3_mprintf( + "PRAGMA key = %Q; SELECT COUNT(*) FROM sqlite_master", + passphrase.c_str()), + sqlite3_free}; + if (!sql || + sqlite3_exec(raw, sql.get(), nullptr, nullptr, nullptr) != SQLITE_OK) + throw unsupported_database{ + "The file `" + path + + "` is not a SQLCipher database that the given passphrase opens"}; + + return sqlite::database{connection}; +} + +} // namespace djinterop::util diff --git a/src/djinterop/util/sqlcipher.hpp b/src/djinterop/util/sqlcipher.hpp new file mode 100644 index 0000000..d85bdc2 --- /dev/null +++ b/src/djinterop/util/sqlcipher.hpp @@ -0,0 +1,41 @@ +/* + This file is part of libdjinterop. + + libdjinterop is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libdjinterop is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with libdjinterop. If not, see . + */ + +#pragma once + +#include + +#include + +namespace djinterop::util +{ +// Exactly one implementation of this header is compiled in: `sqlcipher.cpp` +// where the build has SQLCipher, and otherwise `sqlcipher_unsupported.cpp`. + +/// Open a SQLCipher database for reading. +/// +/// The connection serves the decrypted contents, including anything still in +/// the write-ahead log. Reading a write-ahead-logged database creates the +/// `-shm` and `-wal` files beside it if they are missing, so the directory +/// holding it must be writable. +/// +/// \throws djinterop::unsupported_database If this build cannot open +/// encrypted databases, or the passphrase does not open the file. +[[nodiscard]] sqlite::database open_encrypted_database( + const std::string& path, const std::string& passphrase); + +} // namespace djinterop::util diff --git a/src/djinterop/onelibrary/loader.hpp b/src/djinterop/util/sqlcipher_unsupported.cpp similarity index 64% rename from src/djinterop/onelibrary/loader.hpp rename to src/djinterop/util/sqlcipher_unsupported.cpp index cb4520b..c32b00b 100644 --- a/src/djinterop/onelibrary/loader.hpp +++ b/src/djinterop/util/sqlcipher_unsupported.cpp @@ -15,19 +15,18 @@ along with libdjinterop. If not, see . */ -#pragma once +#include "sqlcipher.hpp" -#include -#include +#include -namespace djinterop::onelibrary +namespace djinterop::util { -struct onelibrary_context; - -/// Decrypt the database on a device and check that it is one. -/// -/// \param path Either the root directory of a device, or the database file. -std::shared_ptr load_context( - const std::string& path, const std::string& passphrase); +sqlite::database open_encrypted_database( + const std::string& path, const std::string&) +{ + throw unsupported_database{ + "This build of libdjinterop cannot read the encrypted database `" + + path + "`, as it was built without EXPERIMENTAL_ENABLE_SQLCIPHER"}; +} -} // namespace djinterop::onelibrary +} // namespace djinterop::util diff --git a/src/djinterop/util/sqlite_query.hpp b/src/djinterop/util/sqlite_query.hpp index 506f644..0352d50 100644 --- a/src/djinterop/util/sqlite_query.hpp +++ b/src/djinterop/util/sqlite_query.hpp @@ -42,11 +42,11 @@ template std::optional first_id( sqlite::database& db, const char* sql, const Args&... args) { - std::optional result; - auto query = db << sql; - ((query << args), ...); - query >> [&](int64_t id) { result = id; }; - return result; + const auto ids = collect_ids(db, sql, args...); + if (ids.empty()) + return std::nullopt; + + return ids.front(); } /// Test whether a query matches any row at all. diff --git a/test/djinterop/onelibrary/content_table_test.cpp b/test/djinterop/onelibrary/content_table_test.cpp index e5cf835..1a74c9c 100644 --- a/test/djinterop/onelibrary/content_table_test.cpp +++ b/test/djinterop/onelibrary/content_table_test.cpp @@ -46,8 +46,7 @@ namespace std::shared_ptr make_context() { sqlite::database db{":memory:"}; - for (const auto& statement : onelibrary_schema_statements()) - db << statement; + create_onelibrary_schema(db.connection().get()); db << "INSERT INTO artist VALUES (1, 'Aphex Twin', ''), " "(2, 'Squarepusher', '')"; diff --git a/test/djinterop/onelibrary/database_test.cpp b/test/djinterop/onelibrary/database_test.cpp index 7271705..9b6b941 100644 --- a/test/djinterop/onelibrary/database_test.cpp +++ b/test/djinterop/onelibrary/database_test.cpp @@ -18,11 +18,8 @@ #define BOOST_TEST_MODULE onelibrary_database_test #include -#include #include -#include #include -#include #include @@ -30,84 +27,18 @@ #include #include -#include "../../../src/djinterop/util/filesystem.hpp" #include "../boost_test_printable.hpp" -#include "../sqlcipher_encryptor.hpp" #include "../temporary_directory.hpp" #include "onelibrary_schema.hpp" namespace utf = boost::unit_test; namespace ol = djinterop::onelibrary; namespace olv1 = djinterop::onelibrary::v1; -namespace crypto = djinterop::util::crypto; namespace { const std::string passphrase = "a passphrase for the fixture"; -/// The encryptor that every fixture is written with. -/// -/// Key derivation is deliberately expensive -- the format stretches the -/// passphrase 256,000 times -- so one salt, and hence one derived key, is -/// shared by every fixture rather than made afresh for each. A database -/// carries its own salt, so nothing about reading one depends on this. -const sqlcipher_encryptor& fixture_encryptor() -{ - static const sqlcipher_encryptor encryptor{passphrase}; - return encryptor; -} - -/// Encrypt a plain SQLite file in place, as SQLCipher would have written it. -/// -/// The fixture is built as an ordinary database and encrypted afterwards, -/// which is the only direction the library offers: it decrypts a device to -/// read it, and never writes one. -void encrypt_in_place(const std::string& path) -{ - std::vector plain; - { - std::ifstream file{path, std::ios::binary}; - plain.assign( - std::istreambuf_iterator{file}, - std::istreambuf_iterator{}); - } - - const crypto::sqlcipher_parameters params; - BOOST_REQUIRE(!plain.empty()); - BOOST_REQUIRE_EQUAL(plain.size() % params.page_size, 0u); - - const auto& encryptor = fixture_encryptor(); - - std::vector encrypted(plain.size()); - for (size_t index = 0; index < plain.size() / params.page_size; ++index) - { - encryptor.encrypt_page( - static_cast(index + 1), - plain.data() + (index * params.page_size), - encrypted.data() + (index * params.page_size)); - } - - std::ofstream{path, std::ios::binary | std::ios::trunc}.write( - reinterpret_cast(encrypted.data()), - static_cast(encrypted.size())); -} - -/// Prepare a plain database to hold pages of the shape SQLCipher expects. -void prepare_plain_database(sqlite3* db) -{ - const crypto::sqlcipher_parameters params; - char* error = nullptr; - const auto sql = "PRAGMA page_size = " + std::to_string(params.page_size); - BOOST_REQUIRE_EQUAL( - sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &error), SQLITE_OK); - sqlite3_free(error); - - auto reserve = static_cast(params.reserve); - BOOST_REQUIRE_EQUAL( - sqlite3_file_control(db, "main", SQLITE_FCNTL_RESERVE_BYTES, &reserve), - SQLITE_OK); -} - void execute(sqlite3* db, const std::string& sql) { char* error = nullptr; @@ -118,10 +49,29 @@ void execute(sqlite3* db, const std::string& sql) rc == SQLITE_OK, "failed to run \"" << sql << "\": " << message); } -void create_onelibrary_schema(sqlite3* db) +/// Create a database encrypted as rekordbox encrypts one. +/// +/// The format is SQLCipher 4 with its default parameters, so the key is all +/// there is to set. +sqlite3* create_encrypted_database(const std::string& path) { - for (const auto& statement : onelibrary_schema_statements()) - execute(db, statement); + sqlite3* db = nullptr; + BOOST_REQUIRE_EQUAL( + sqlite3_open_v2( + path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + nullptr), + SQLITE_OK); + execute(db, "PRAGMA key = '" + passphrase + "'"); + return db; +} + +/// Create the directories of a device, returning where its database goes. +std::string make_device_directories(const std::string& root) +{ + const auto path = root + "/" + ol::database_relative_path; + boost::filesystem::create_directories( + boost::filesystem::path{path}.parent_path()); + return path; } /// Build a device holding a small OneLibrary export. @@ -133,19 +83,9 @@ void create_onelibrary_schema(sqlite3* db) std::string make_device(const temporary_directory& temp_dir) { const auto root = temp_dir.temp_dir; - djinterop::util::create_dir(root + "/PIONEER"); - djinterop::util::create_dir(root + "/PIONEER/rekordbox"); - - const auto path = root + "/" + ol::database_relative_path; - - sqlite3* db = nullptr; - BOOST_REQUIRE_EQUAL( - sqlite3_open_v2( - path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, - nullptr), - SQLITE_OK); + const auto path = make_device_directories(root); - prepare_plain_database(db); + auto* db = create_encrypted_database(path); create_onelibrary_schema(db); @@ -201,7 +141,6 @@ std::string make_device(const temporary_directory& temp_dir) "'2026-01-01', 0, 0)"); sqlite3_close(db); - encrypt_in_place(path); return root; } @@ -224,14 +163,6 @@ const shared_device& device_fixture() return instance; } -/// The shared device, opened once. -djinterop::database loaded_database() -{ - static const djinterop::database db = - ol::load_database(device_fixture().path, passphrase); - return db; -} - /// The shared device, opened once as a library. const olv1::library& loaded_library() { @@ -239,6 +170,12 @@ const olv1::library& loaded_library() return lib; } +/// The shared device, through the same connection as `loaded_library()`. +djinterop::database loaded_database() +{ + return loaded_library().database(); +} + } // anonymous namespace BOOST_TEST_DECORATOR(*utf::description( @@ -504,13 +441,7 @@ BOOST_AUTO_TEST_CASE( temporary_directory temp_dir; const auto path = temp_dir.temp_dir + "/deep.db"; - sqlite3* db = nullptr; - BOOST_REQUIRE_EQUAL( - sqlite3_open_v2( - path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, - nullptr), - SQLITE_OK); - prepare_plain_database(db); + auto* db = create_encrypted_database(path); create_onelibrary_schema(db); // Every export carries a `property` row, whatever else it holds. @@ -528,12 +459,7 @@ BOOST_AUTO_TEST_CASE( "INSERT INTO playlist VALUES (1, 1, 'Root', 0, 0, 0), " "(2, 1, 'Middle A', 0, 0, 1), (3, 2, 'Middle B', 0, 0, 1), " "(4, 1, 'Leaf A', 0, 0, 2)"); - execute( - db, - "INSERT INTO property VALUES ('FIXTURE', '1000', 0, " - "'2026-01-01', 0, 0)"); sqlite3_close(db); - encrypt_in_place(path); auto loaded = ol::load_database(path, passphrase); const auto root = loaded.root_crate_by_name("Root"); @@ -555,18 +481,12 @@ BOOST_AUTO_TEST_CASE(load_database__a_checkpointed_log__is_read) { // A real export is written in WAL mode and checkpointed on eject, which // removes the log but leaves the header declaring the database - // write-ahead-logged. SQLite refuses to open one of those read-only - // without the log, so the header has to be rewritten as it is decrypted. + // write-ahead-logged. SQLite will not open one of those read-only unless + // it can create the log again. temporary_directory temp_dir; const auto path = temp_dir.temp_dir + "/checkpointed.db"; - sqlite3* db = nullptr; - BOOST_REQUIRE_EQUAL( - sqlite3_open_v2( - path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, - nullptr), - SQLITE_OK); - prepare_plain_database(db); + auto* db = create_encrypted_database(path); create_onelibrary_schema(db); execute(db, "PRAGMA journal_mode = WAL"); execute( @@ -583,8 +503,6 @@ BOOST_AUTO_TEST_CASE(load_database__a_checkpointed_log__is_read) sqlite3_close(db); std::remove((path + "-wal").c_str()); - encrypt_in_place(path); - auto loaded = ol::load_database(path, passphrase); const auto tracks = loaded.tracks(); BOOST_REQUIRE_EQUAL(tracks.size(), 1u); @@ -598,131 +516,14 @@ BOOST_AUTO_TEST_CASE(verify__a_database_missing_its_tables__is_rejected) temporary_directory temp_dir; const auto path = temp_dir.temp_dir + "/not-onelibrary.db"; - sqlite3* db = nullptr; - BOOST_REQUIRE_EQUAL( - sqlite3_open_v2( - path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, - nullptr), - SQLITE_OK); - prepare_plain_database(db); + auto* db = create_encrypted_database(path); execute(db, "CREATE TABLE something_else(id INTEGER PRIMARY KEY)"); sqlite3_close(db); - encrypt_in_place(path); BOOST_CHECK_THROW( ol::load_database(path, passphrase), djinterop::database_inconsistency); } -namespace -{ -/// Read a whole file. -std::vector read_file(const std::string& path) -{ - std::ifstream file{path, std::ios::binary}; - return std::vector{ - std::istreambuf_iterator{file}, std::istreambuf_iterator{}}; -} - -void write_file(const std::string& path, const std::vector& data) -{ - std::ofstream{path, std::ios::binary | std::ios::trunc}.write( - reinterpret_cast(data.data()), - static_cast(data.size())); -} - -uint32_t load_be32(const uint8_t* p) -{ - return (static_cast(p[0]) << 24) | - (static_cast(p[1]) << 16) | - (static_cast(p[2]) << 8) | static_cast(p[3]); -} - -void store_be32(uint8_t* p, uint32_t value) -{ - p[0] = static_cast(value >> 24); - p[1] = static_cast(value >> 16); - p[2] = static_cast(value >> 8); - p[3] = static_cast(value); -} - -/// Encrypt a plain database and its write-ahead log the way SQLCipher does. -/// -/// SQLCipher encrypts the page inside each log frame, and SQLite checksums the -/// frame over the bytes as they end up in the file -- so over the ciphertext. -/// Reproducing that here is what makes this a test of reading a real log -/// rather than of reading one this library made up. -void encrypt_database_and_log( - const std::string& path, const sqlcipher_encryptor& encryptor) -{ - const auto& params = encryptor.params(); - - auto plain = read_file(path); - BOOST_REQUIRE_EQUAL(plain.size() % params.page_size, 0u); - std::vector encrypted(plain.size()); - for (size_t index = 0; index < plain.size() / params.page_size; ++index) - encryptor.encrypt_page( - static_cast(index + 1), - plain.data() + (index * params.page_size), - encrypted.data() + (index * params.page_size)); - write_file(path, encrypted); - - auto log = read_file(path + "-wal"); - BOOST_REQUIRE_GE(log.size(), 32u); - - // The checksums of the log are computed over its contents in the byte - // order its magic number selects, and stored big-endian. - const auto big_endian = (load_be32(log.data()) & 1) != 0; - const auto read_word = [&](const uint8_t* p) - { - if (big_endian) - return load_be32(p); - - return static_cast( - (static_cast(p[3]) << 24) | - (static_cast(p[2]) << 16) | - (static_cast(p[1]) << 8) | static_cast(p[0])); - }; - - uint32_t s0 = 0; - uint32_t s1 = 0; - const auto accumulate = [&](const uint8_t* data, size_t length) - { - for (size_t offset = 0; offset + 8 <= length; offset += 8) - { - s0 += read_word(data + offset) + s1; - s1 += read_word(data + offset + 4) + s0; - } - }; - - accumulate(log.data(), 24); - BOOST_REQUIRE_EQUAL(s0, load_be32(log.data() + 24)); - BOOST_REQUIRE_EQUAL(s1, load_be32(log.data() + 28)); - - const auto frame_length = 24 + params.page_size; - size_t frames = 0; - for (size_t offset = 32; offset + frame_length <= log.size(); - offset += frame_length) - { - auto* frame = log.data() + offset; - const auto page_number = load_be32(frame); - - std::vector page(params.page_size); - encryptor.encrypt_page(page_number, frame + 24, page.data()); - std::memcpy(frame + 24, page.data(), page.size()); - - accumulate(frame, 8); - accumulate(frame + 24, params.page_size); - store_be32(frame + 16, s0); - store_be32(frame + 20, s1); - ++frames; - } - - BOOST_REQUIRE_GT(frames, 0u); - write_file(path + "-wal", log); -} - -} // anonymous namespace - BOOST_TEST_DECORATOR( *utf::description("load_database() reads data left in the write-ahead log")) BOOST_AUTO_TEST_CASE(load_database__data_left_in_the_log__is_read) @@ -732,19 +533,11 @@ BOOST_AUTO_TEST_CASE(load_database__data_left_in_the_log__is_read) // library, with no error at all. temporary_directory temp_dir; const auto device = temp_dir.temp_dir + "/device"; - djinterop::util::create_dir(device); - djinterop::util::create_dir(device + "/PIONEER"); - djinterop::util::create_dir(device + "/PIONEER/rekordbox"); + const auto path = make_device_directories(device); - const auto path = device + "/" + std::string{ol::database_relative_path}; - - sqlite3* db = nullptr; - BOOST_REQUIRE_EQUAL( - sqlite3_open_v2( - path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, - nullptr), - SQLITE_OK); - prepare_plain_database(db); + // The database is written beside the device, and copied onto it. + const auto work = temp_dir.temp_dir + "/work.db"; + auto* db = create_encrypted_database(work); create_onelibrary_schema(db); execute( db, @@ -763,18 +556,12 @@ BOOST_AUTO_TEST_CASE(load_database__data_left_in_the_log__is_read) "INSERT INTO content (content_id, title, path) " "VALUES (2, 'In The Log', '/b.mp3')"); - // Copy the pair aside while the connection is open, because closing it - // would fold the log back in. - const auto db_copy = temp_dir.temp_dir + "/copy.db"; - write_file(db_copy, read_file(path)); - write_file(db_copy + "-wal", read_file(path + "-wal")); + // Copy the pair while the connection is open, because closing it would + // fold the log back in. + boost::filesystem::copy_file(work, path); + boost::filesystem::copy_file(work + "-wal", path + "-wal"); sqlite3_close(db); - write_file(path, read_file(db_copy)); - write_file(path + "-wal", read_file(db_copy + "-wal")); - - encrypt_database_and_log(path, fixture_encryptor()); - auto loaded = ol::load_database(device, passphrase); const auto tracks = loaded.tracks(); BOOST_REQUIRE_EQUAL(tracks.size(), 2u); diff --git a/test/djinterop/onelibrary/onelibrary_schema.hpp b/test/djinterop/onelibrary/onelibrary_schema.hpp index 447b535..fae8c61 100644 --- a/test/djinterop/onelibrary/onelibrary_schema.hpp +++ b/test/djinterop/onelibrary/onelibrary_schema.hpp @@ -21,60 +21,33 @@ #include #include #include -#include + +#include #define ONELIBRARY_STRINGIFY(x) ONELIBRARY_STRINGIFY_(x) #define ONELIBRARY_STRINGIFY_(x) #x -/// The schema a real OneLibrary export carries, read from the reference +/// Create the tables a real OneLibrary export carries, from the reference /// script in `testdata/ref/onelibrary`. /// /// The script is the record of what a device holds; see its own comments. -inline const std::vector& onelibrary_schema_statements() +inline void create_onelibrary_schema(sqlite3* db) { - static const std::vector statements = [] - { - const std::string path = - std::string{ONELIBRARY_STRINGIFY(TESTDATA_DIR)} + - "/ref/onelibrary/schema.sql"; - - std::ifstream file{path}; - if (!file) - throw std::runtime_error{"Cannot read the schema at " + path}; - - std::ostringstream contents; - contents << file.rdbuf(); - - // The script is a sequence of statements separated by semicolons, and - // nothing in it holds one in a string literal. - std::vector result; - std::string statement; - for (const auto character : contents.str()) - { - if (character != ';') - { - statement += character; - continue; - } - - // Comment lines belong to the script, not to the statement. - std::string stripped; - std::istringstream lines{statement}; - for (std::string line; std::getline(lines, line);) - if (line.rfind("--", 0) != 0) - stripped += line + " "; - - const auto begin = stripped.find_first_not_of(" \t\r\n"); - const auto end = stripped.find_last_not_of(" \t\r\n"); - if (begin != std::string::npos) - result.push_back( - stripped.substr(begin, end - begin + 1)); - - statement.clear(); - } - - return result; - }(); - - return statements; + const std::string path = std::string{ONELIBRARY_STRINGIFY(TESTDATA_DIR)} + + "/ref/onelibrary/schema.sql"; + + std::ifstream file{path}; + if (!file) + throw std::runtime_error{"Cannot read the schema at " + path}; + + std::ostringstream script; + script << file.rdbuf(); + + char* error = nullptr; + const auto rc = + sqlite3_exec(db, script.str().c_str(), nullptr, nullptr, &error); + const std::string message = error != nullptr ? error : ""; + sqlite3_free(error); + if (rc != SQLITE_OK) + throw std::runtime_error{"Cannot create the schema: " + message}; } diff --git a/test/djinterop/onelibrary/playlist_table_test.cpp b/test/djinterop/onelibrary/playlist_table_test.cpp index 8d30c53..fee0638 100644 --- a/test/djinterop/onelibrary/playlist_table_test.cpp +++ b/test/djinterop/onelibrary/playlist_table_test.cpp @@ -46,8 +46,7 @@ namespace std::shared_ptr make_context() { sqlite::database db{":memory:"}; - for (const auto& statement : onelibrary_schema_statements()) - db << statement; + create_onelibrary_schema(db.connection().get()); db << "INSERT INTO playlist (playlist_id, sequenceNo, name, " "playlist_id_parent) VALUES (1, 1, 'Sets', NULL), " diff --git a/test/djinterop/sqlcipher_encryptor.hpp b/test/djinterop/sqlcipher_encryptor.hpp deleted file mode 100644 index b54f7b2..0000000 --- a/test/djinterop/sqlcipher_encryptor.hpp +++ /dev/null @@ -1,134 +0,0 @@ -/* - This file is part of libdjinterop. - - libdjinterop is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libdjinterop is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with libdjinterop. If not, see . - */ - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "../../src/djinterop/util/crypto/aes.hpp" -#include "../../src/djinterop/util/crypto/sha512.hpp" -#include "../../src/djinterop/util/crypto/sqlcipher_codec.hpp" - -/// Writes SQLCipher 4 pages, so that a test can build a database for the -/// library to read. -/// -/// The library only ever decrypts, so the encrypting direction lives here -/// rather than beside it. Writing the pages from the format description -/// independently also makes a round trip a check of the format, rather than a -/// check that the library agrees with itself. -class sqlcipher_encryptor -{ -public: - using parameters = djinterop::util::crypto::sqlcipher_parameters; - using salt_type = djinterop::util::crypto::sqlcipher_salt; - - explicit sqlcipher_encryptor( - const std::string& passphrase, const parameters& params = {}) : - params_{params}, - salt_{random_bytes()}, - page_key_{djinterop::util::crypto::pbkdf2_hmac_sha512( - reinterpret_cast(passphrase.data()), - passphrase.size(), salt_.data(), salt_.size(), - params.kdf_iterations, djinterop::util::crypto::aes256_key_length)}, - hmac_key_{derive_hmac_key()}, cipher_{page_key_.data()} - { - } - - /// The parameters the pages are written under. - [[nodiscard]] const parameters& params() const noexcept { return params_; } - - /// The salt written into the first page, with which a codec that is to - /// read the database has to be built. - [[nodiscard]] const salt_type& salt() const noexcept { return salt_; } - - /// Encrypt one page, of `page_size` bytes, under a fresh random vector. - void encrypt_page( - uint32_t page_number, const uint8_t* decrypted, - uint8_t* encrypted) const - { - using namespace djinterop::util::crypto; - - // Page one carries the salt where a plain SQLite file has its magic, - // so its ciphertext starts after it. - const auto offset = page_number == 1 ? sqlcipher_salt_length : 0; - const auto length = params_.payload_size() - offset; - const auto iv = random_bytes>(); - - cipher_.encrypt( - iv.data(), decrypted + offset, encrypted + offset, length); - - if (page_number == 1) - std::memcpy(encrypted, salt_.data(), salt_.size()); - - // The tag covers ciphertext || IV || page number, the last as a - // little-endian 32-bit integer, which binds a page to its position. - std::vector message; - message.insert( - message.end(), encrypted + offset, encrypted + offset + length); - message.insert(message.end(), iv.begin(), iv.end()); - for (int shift : {0, 8, 16, 24}) - message.push_back(static_cast(page_number >> shift)); - - const auto tag = hmac_sha512( - hmac_key_.data(), hmac_key_.size(), message.data(), message.size()); - - auto* reserve = encrypted + params_.payload_size(); - std::memcpy(reserve, iv.data(), iv.size()); - std::memcpy(reserve + iv.size(), tag.data(), tag.size()); - - const auto used = iv.size() + tag.size(); - if (params_.reserve > used) - std::memset(reserve + used, 0, params_.reserve - used); - } - -private: - template static Bytes random_bytes() - { - static std::random_device rng; - static std::uniform_int_distribution byte_dist{0, 255}; - - Bytes bytes{}; - for (auto& byte : bytes) - byte = static_cast(byte_dist(rng)); - - return bytes; - } - - /// The tag key is derived from the page key, under the salt with every - /// byte flipped by 0x3a, as SQLCipher does it. - [[nodiscard]] std::vector derive_hmac_key() const - { - salt_type hmac_salt{}; - for (size_t index = 0; index < salt_.size(); ++index) - hmac_salt[index] = static_cast(salt_[index] ^ 0x3a); - - return djinterop::util::crypto::pbkdf2_hmac_sha512( - page_key_.data(), page_key_.size(), hmac_salt.data(), - hmac_salt.size(), params_.hmac_kdf_iterations, 32); - } - - parameters params_; - salt_type salt_; - std::vector page_key_; - std::vector hmac_key_; - djinterop::util::crypto::aes256_cbc cipher_; -}; diff --git a/test/djinterop/util/crypto_test.cpp b/test/djinterop/util/crypto_test.cpp deleted file mode 100644 index 33d09cb..0000000 --- a/test/djinterop/util/crypto_test.cpp +++ /dev/null @@ -1,398 +0,0 @@ -/* - This file is part of libdjinterop. - - libdjinterop is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libdjinterop is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with libdjinterop. If not, see . - */ - -#define BOOST_TEST_MODULE crypto_test -#include - -#include -#include -#include - -#include "../../../src/djinterop/util/crypto/aes.hpp" -#include "../../../src/djinterop/util/crypto/sha512.hpp" -#include "../../../src/djinterop/util/crypto/sqlcipher_codec.hpp" -#include "../sqlcipher_encryptor.hpp" - -namespace utf = boost::unit_test; - -using namespace djinterop::util::crypto; - -namespace -{ -std::string to_hex(const uint8_t* data, size_t length) -{ - static const char* digits = "0123456789abcdef"; - std::string out; - out.reserve(length * 2); - for (size_t i = 0; i < length; ++i) - { - out.push_back(digits[data[i] >> 4]); - out.push_back(digits[data[i] & 0x0f]); - } - return out; -} - -const uint8_t* bytes_of(const std::string& s) -{ - return reinterpret_cast(s.data()); -} - -std::vector from_hex(const std::string& hex) -{ - std::vector out; - out.reserve(hex.size() / 2); - for (size_t i = 0; i + 1 < hex.size(); i += 2) - { - out.push_back( - static_cast(std::stoul(hex.substr(i, 2), nullptr, 16))); - } - - return out; -} - -/// Both ways the cipher can do its work: a machine with AES instructions would -/// otherwise never reach the tables, and one without never the instructions, so -/// every test of the cipher runs over both. -const aes_implementation implementations[] = { - aes_implementation::automatic, aes_implementation::tabulated}; - -/// Parameters with a cheap key derivation. -/// -/// The real format stretches the passphrase 256,000 times, which is the point -/// of it, but paying that in every test makes the suite slow to no purpose, -/// especially in an unoptimised build. Tests about the page format use these; -/// the tests about key derivation itself use the real ones. -sqlcipher_parameters cheap_parameters() -{ - sqlcipher_parameters params; - params.kdf_iterations = 1000; - return params; -} - -} // anonymous namespace - -BOOST_TEST_DECORATOR(*utf::description("sha512 matches the published vectors")) -BOOST_AUTO_TEST_CASE(sha512__known_vectors__match) -{ - // FIPS 180-4 test vectors. - const std::string abc = "abc"; - const auto digest = sha512::hash(bytes_of(abc), abc.size()); - BOOST_CHECK_EQUAL( - to_hex(digest.data(), digest.size()), - "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a" - "2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f"); - - const std::string empty; - const auto empty_digest = sha512::hash(bytes_of(empty), 0); - BOOST_CHECK_EQUAL( - to_hex(empty_digest.data(), empty_digest.size()), - "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce" - "47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e"); -} - -BOOST_TEST_DECORATOR( - *utf::description("sha512 streamed in chunks matches one shot")) -BOOST_AUTO_TEST_CASE(sha512__streamed_in_chunks__matches_one_shot) -{ - const std::string input(1000, 'x'); - const auto expected = sha512::hash(bytes_of(input), input.size()); - - // Chunk sizes that divide neither the block length nor each other, so the - // buffering path is exercised at every alignment. - sha512 streamed; - for (size_t offset = 0; offset < input.size(); offset += 7) - streamed.update( - bytes_of(input) + offset, - std::min(7, input.size() - offset)); - - BOOST_CHECK(streamed.finalise() == expected); -} - -BOOST_TEST_DECORATOR( - *utf::description("hmac_sha512 matches the RFC 4231 vector")) -BOOST_AUTO_TEST_CASE(hmac_sha512__rfc_4231_vector__matches) -{ - const std::vector key(20, 0x0b); - const std::string data = "Hi There"; - const auto tag = - hmac_sha512(key.data(), key.size(), bytes_of(data), data.size()); - - BOOST_CHECK_EQUAL( - to_hex(tag.data(), tag.size()), - "87aa7cdea5ef619d4ff0b4241a1d6cb02379f4e2ce4ec2787ad0b30545e17cde" - "daa833b7d6b8a702038b274eaea3f4e4be9d914eeb61f1702e696c203a126854"); -} - -BOOST_TEST_DECORATOR( - *utf::description("pbkdf2_hmac_sha512 matches the published vector")) -BOOST_AUTO_TEST_CASE(pbkdf2_hmac_sha512__known_vector__matches) -{ - const std::string password = "passwd"; - const std::string salt = "salt"; - const auto derived = pbkdf2_hmac_sha512( - bytes_of(password), password.size(), bytes_of(salt), salt.size(), 1, - 64); - - BOOST_CHECK_EQUAL( - to_hex(derived.data(), derived.size()), - "c74319d99499fc3e9013acff597c23c5baf0a0bec5634c46b8352b793e324723" - "d55caa76b2b25c43402dcfdc06cdcf66f95b7d0429420b39520006749c51a04e"); -} - -BOOST_TEST_DECORATOR(*utf::description( - "pbkdf2_hmac_sha512 spans several blocks for a long output")) -BOOST_AUTO_TEST_CASE(pbkdf2_hmac_sha512__long_output__spans_several_blocks) -{ - // More than one digest of output, so the block-index loop is exercised. - const std::string password = "passwd"; - const std::string salt = "salt"; - const auto derived = pbkdf2_hmac_sha512( - bytes_of(password), password.size(), bytes_of(salt), salt.size(), 1, - 100); - - BOOST_REQUIRE_EQUAL(derived.size(), 100u); - - // Its first 64 bytes are the same derivation as above. - const auto shorter = pbkdf2_hmac_sha512( - bytes_of(password), password.size(), bytes_of(salt), salt.size(), 1, - 64); - BOOST_CHECK( - std::vector(derived.begin(), derived.begin() + 64) == shorter); -} - -BOOST_TEST_DECORATOR(*utf::description("aes256 matches the FIPS 197 vector")) -BOOST_AUTO_TEST_CASE(aes256__fips_197_vector__matches) -{ - uint8_t key[aes256_key_length]; - for (size_t i = 0; i < sizeof(key); ++i) - key[i] = static_cast(i); - - const uint8_t plaintext[aes_block_length] = { - 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, - 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}; - const uint8_t zero_iv[aes_block_length] = {}; - - for (const auto implementation : implementations) - { - // A single block under a zero vector is plain ECB, which is what the - // published vector covers. - uint8_t ciphertext[aes_block_length]; - const aes256_cbc cipher{key, implementation}; - cipher.encrypt(zero_iv, plaintext, ciphertext, sizeof(plaintext)); - BOOST_CHECK_EQUAL( - to_hex(ciphertext, sizeof(ciphertext)), - "8ea2b7ca516745bfeafc49904b496089"); - - uint8_t recovered[aes_block_length]; - cipher.decrypt(zero_iv, ciphertext, recovered, sizeof(ciphertext)); - BOOST_CHECK_EQUAL( - to_hex(recovered, sizeof(recovered)), - to_hex(plaintext, sizeof(plaintext))); - } -} - -BOOST_TEST_DECORATOR( - *utf::description("aes256_cbc matches the NIST SP 800-38A vector")) -BOOST_AUTO_TEST_CASE(aes256_cbc__sp_800_38a_vector__matches) -{ - // F.2.5 and F.2.6 chain four blocks, and so pin the mode itself rather - // than only the block cipher under it. - const auto key = from_hex( - "603deb1015ca71be2b73aef0857d7781" - "1f352c073b6108d72d9810a30914dff4"); - const auto iv = from_hex("000102030405060708090a0b0c0d0e0f"); - const auto plaintext = from_hex( - "6bc1bee22e409f96e93d7e117393172a" - "ae2d8a571e03ac9c9eb76fac45af8e51" - "30c81c46a35ce411e5fbc1191a0a52ef" - "f69f2445df4f9b17ad2b417be66c3710"); - const std::string expected = - "f58c4c04d6e5f1ba779eabfb5f7bfbd6" - "9cfc4e967edb808d679f777bc6702c7d" - "39f23369a9d9bacfa530e26304231461" - "b2eb05e2c39be9fcda6c19078c6a9d1b"; - - for (const auto implementation : implementations) - { - const aes256_cbc cipher{key.data(), implementation}; - - std::vector ciphertext(plaintext.size()); - cipher.encrypt( - iv.data(), plaintext.data(), ciphertext.data(), plaintext.size()); - BOOST_CHECK_EQUAL( - to_hex(ciphertext.data(), ciphertext.size()), expected); - - std::vector recovered(ciphertext.size()); - cipher.decrypt( - iv.data(), ciphertext.data(), recovered.data(), ciphertext.size()); - BOOST_CHECK(recovered == plaintext); - } -} - -BOOST_TEST_DECORATOR( - *utf::description("aes256_cbc round-trips multiple blocks")) -BOOST_AUTO_TEST_CASE(aes256_cbc__multiple_blocks__round_trip) -{ - uint8_t key[aes256_key_length]; - uint8_t iv[aes_block_length]; - for (size_t i = 0; i < sizeof(key); ++i) - key[i] = static_cast(i * 3); - for (size_t i = 0; i < sizeof(iv); ++i) - iv[i] = static_cast(0xa0 + i); - - std::vector plaintext(512); - for (size_t i = 0; i < plaintext.size(); ++i) - plaintext[i] = static_cast(i * 7); - - for (const auto implementation : implementations) - { - const aes256_cbc cipher{key, implementation}; - - std::vector ciphertext(plaintext.size()); - cipher.encrypt( - iv, plaintext.data(), ciphertext.data(), plaintext.size()); - BOOST_CHECK(ciphertext != plaintext); - - // Decryption in place must work, as the codec relies on it. - std::vector buffer = ciphertext; - cipher.decrypt(iv, buffer.data(), buffer.data(), buffer.size()); - BOOST_CHECK(buffer == plaintext); - } -} - -BOOST_TEST_DECORATOR(*utf::description( - "aes256_cbc decrypts every length the same way, whichever implementation")) -BOOST_AUTO_TEST_CASE(aes256_cbc__every_length__agrees_across_implementations) -{ - uint8_t key[aes256_key_length]; - uint8_t iv[aes_block_length]; - for (size_t i = 0; i < sizeof(key); ++i) - key[i] = static_cast(0x5a + i); - for (size_t i = 0; i < sizeof(iv); ++i) - iv[i] = static_cast(i * 11); - - std::vector ciphertext(24 * aes_block_length); - for (size_t i = 0; i < ciphertext.size(); ++i) - ciphertext[i] = static_cast((i * 31) ^ 0x9c); - - const aes256_cbc hardware{key, aes_implementation::automatic}; - const aes256_cbc tabulated{key, aes_implementation::tabulated}; - - // Decryption runs several blocks at a time where the processor allows it, - // so lengths that do not divide by that group size exercise the tail. - for (size_t blocks = 0; blocks <= 24; ++blocks) - { - const auto length = blocks * aes_block_length; - - std::vector by_hardware(length); - hardware.decrypt(iv, ciphertext.data(), by_hardware.data(), length); - - std::vector by_tables(length); - tabulated.decrypt(iv, ciphertext.data(), by_tables.data(), length); - - BOOST_CHECK(by_hardware == by_tables); - } -} - -BOOST_TEST_DECORATOR( - *utf::description("sqlcipher_codec round-trips a page losslessly")) -BOOST_AUTO_TEST_CASE(sqlcipher_codec__page_round_trip__is_lossless) -{ - const auto params = cheap_parameters(); - const sqlcipher_encryptor writer{"a passphrase", params}; - const sqlcipher_codec codec{"a passphrase", writer.salt(), params}; - - std::vector page(params.page_size, 0); - for (size_t i = 0; i < params.payload_size(); ++i) - page[i] = static_cast((i * 31) & 0xff); - - for (uint32_t page_number : {1u, 2u, 4096u}) - { - std::vector encrypted(params.page_size); - writer.encrypt_page(page_number, page.data(), encrypted.data()); - - std::vector decrypted(params.page_size); - codec.decrypt_page(page_number, encrypted.data(), decrypted.data()); - - // Page one carries the salt in place of the first sixteen bytes. - const auto offset = page_number == 1 ? sqlcipher_salt_length : 0; - BOOST_CHECK( - std::memcmp( - decrypted.data() + offset, page.data() + offset, - params.payload_size() - offset) == 0); - } -} - -BOOST_TEST_DECORATOR( - *utf::description("sqlcipher_codec rejects a tampered page")) -BOOST_AUTO_TEST_CASE(sqlcipher_codec__tampered_page__is_rejected) -{ - const auto params = cheap_parameters(); - const sqlcipher_encryptor writer{"a passphrase", params}; - const sqlcipher_codec codec{"a passphrase", writer.salt(), params}; - - std::vector page(params.page_size, 0x5a); - std::vector encrypted(params.page_size); - writer.encrypt_page(2, page.data(), encrypted.data()); - - encrypted[100] = static_cast(encrypted[100] ^ 0x01); - - std::vector decrypted(params.page_size); - BOOST_CHECK_THROW( - codec.decrypt_page(2, encrypted.data(), decrypted.data()), - sqlcipher_error); -} - -BOOST_TEST_DECORATOR( - *utf::description("sqlcipher_codec rejects a wrong passphrase")) -BOOST_AUTO_TEST_CASE(sqlcipher_codec__wrong_passphrase__is_rejected) -{ - const auto params = cheap_parameters(); - const sqlcipher_encryptor writer{"the right one", params}; - const sqlcipher_codec reader{"the wrong one", writer.salt(), params}; - - std::vector page(params.page_size, 0x11); - std::vector encrypted(params.page_size); - writer.encrypt_page(3, page.data(), encrypted.data()); - - std::vector decrypted(params.page_size); - BOOST_CHECK_THROW( - reader.decrypt_page(3, encrypted.data(), decrypted.data()), - sqlcipher_error); -} - -BOOST_TEST_DECORATOR( - *utf::description("sqlcipher_codec binds the page number into the tag")) -BOOST_AUTO_TEST_CASE(sqlcipher_codec__page_number__is_bound_into_the_tag) -{ - const auto params = cheap_parameters(); - const sqlcipher_encryptor writer{"a passphrase", params}; - const sqlcipher_codec codec{"a passphrase", writer.salt(), params}; - - std::vector page(params.page_size, 0x22); - std::vector encrypted(params.page_size); - writer.encrypt_page(7, page.data(), encrypted.data()); - - std::vector decrypted(params.page_size); - BOOST_CHECK_NO_THROW( - codec.decrypt_page(7, encrypted.data(), decrypted.data())); - - // The same bytes, read as a different page, must not verify. - BOOST_CHECK_THROW( - codec.decrypt_page(8, encrypted.data(), decrypted.data()), - sqlcipher_error); -} From 7846d70b85e89e89d5f3a2de4d9754ab2ea15573 Mon Sep 17 00:00:00 2001 From: Illia Komsa Date: Wed, 23 Sep 2026 18:52:50 +0200 Subject: [PATCH 10/10] Install libdjinterop with SQLCipher so consumers can use it A static build with EXPERIMENTAL_ENABLE_SQLCIPHER could not be installed, as the sqlcipher target it links was in no export set. It is now installed and exported beside DjInterop, and the package config finds OpenSSL for it. The exported include directory also pointed one level too deep, so the installed headers could not be included as . Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 10 +++++++++- DjInteropConfig.cmake.in | 3 +++ ext/sqlcipher/CMakeLists.txt | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 91ccbc3..92abd8d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -265,7 +265,7 @@ target_include_directories( DjInterop PUBLIC $ $ - $) + $) # Always rely on system installation of zlib. set(ZLIB_MIN_VERSION 1.2.8) @@ -345,6 +345,14 @@ endif() set_target_properties(DjInterop PROPERTIES C_VISIBILITY_PRESET hidden) set_target_properties(DjInterop PROPERTIES CXX_VISIBILITY_PRESET hidden) +# A static DjInterop does not carry the SQLCipher library inside it, so that +# library is installed beside it for consumers to link as well. +if(EXPERIMENTAL_ENABLE_SQLCIPHER) + install(TARGETS sqlcipher + EXPORT DjInteropTargets + ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}") +endif() + install(TARGETS DjInterop EXPORT DjInteropTargets ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" diff --git a/DjInteropConfig.cmake.in b/DjInteropConfig.cmake.in index 22d88cf..7690bac 100644 --- a/DjInteropConfig.cmake.in +++ b/DjInteropConfig.cmake.in @@ -14,6 +14,9 @@ endif() if(DJINTEROP_SYSTEM_SQLITE) find_dependency(SQLite3) endif() +if(DJINTEROP_EXPERIMENTAL_ENABLE_SQLCIPHER) + find_dependency(OpenSSL) +endif() include("${CMAKE_CURRENT_LIST_DIR}/DjInteropTargets.cmake") diff --git a/ext/sqlcipher/CMakeLists.txt b/ext/sqlcipher/CMakeLists.txt index 1bd9533..90d170d 100644 --- a/ext/sqlcipher/CMakeLists.txt +++ b/ext/sqlcipher/CMakeLists.txt @@ -27,7 +27,7 @@ target_compile_definitions( target_include_directories( sqlcipher PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}) + $) # SQLCipher needs OpenSSL for AES-256-CBC, PBKDF2-HMAC-SHA512 find_package(OpenSSL REQUIRED)