Refuse to build over an output directory that something else modified

src/build/delta.cc

@@ -7,6 +7,7 @@
 #include <span>          // std::span
 #include <string>        // std::string
 #include <string_view>   // std::string_view
+#include <system_error>  // std::error_code
 #include <unordered_map> // std::unordered_map
 #include <unordered_set> // std::unordered_set
 #include <vector>        // std::vector
@@ -532,8 +533,12 @@ auto delta_engine(const BuildPhase phase, const BuildPlan::Type build_type,
         continue;
       }
 
+      // Probed without throwing, since a global this build cannot examine,
+      // whatever the reason, is one to produce again rather than trust
+      std::error_code existence_error;
       const auto path{output / rule.filename};
-      if (entries.contains(path.native()) && !std::filesystem::exists(path)) {
+      if (entries.contains(path.native()) &&
+          !std::filesystem::exists(path, existence_error)) {
         missing_globals[index] = true;
         has_missing_globals = true;
       }

src/index/index.cc

@@ -245,6 +245,14 @@ Advanced Options:
 
      Set the maximum number of direct entries in a directory listing
 
+Output Directory:
+
+   The output directory is owned by the indexer. Do NOT:
+
+   - Manually modify, add, or delete any file inside it
+   - Sync content into it or partially restore it from a backup
+   - Run concurrent indexer invocations over the same output
+
 For more documentation, visit https://one.sourcemeta.com
 )EOF"};
 
@@ -829,6 +837,13 @@ auto main(int argc, char *argv[]) noexcept -> int {
     std::print(stdout, "error: {}\n  at path {}\n  with count {}\n",
                error.what(), error.path().string(), error.count());
     return EXIT_FAILURE;
+  } catch (const sourcemeta::one::ResolverMissingCachedArtifactError &error) {
+    std::print(stdout,
+               "error: {}\n  at path {}\n\n"
+               "Something other than the indexer modified the output "
+               "directory, so delete it and index again from scratch\n",
+               error.what(), error.path().string());
+    return EXIT_FAILURE;
   } catch (const sourcemeta::one::ResolverNotASchemaError &error) {
     std::print(stdout, "error: {}\n  at path {}\n", error.what(),
                error.path().string());

src/resolver/include/sourcemeta/one/resolver_error.h

@@ -26,6 +26,27 @@ class ResolverNotASchemaError : public std::exception {
   std::filesystem::path path_;
 };
 
+// The output directory belongs to the indexer, so a recorded artifact that is
+// gone from disk means something else modified the directory, which is the one
+// assumption the build cannot recover from on its own
+class ResolverMissingCachedArtifactError : public std::exception {
+public:
+  ResolverMissingCachedArtifactError(std::filesystem::path path)
+      : path_{std::move(path)} {}
+
+  [[nodiscard]] auto what() const noexcept -> const char * override {
+    return "The build state references an artifact that no longer exists on "
+           "disk";
+  }
+
+  [[nodiscard]] auto path() const noexcept -> const std::filesystem::path & {
+    return this->path_;
+  }
+
+private:
+  std::filesystem::path path_;
+};
+
 class ResolverOutsideBaseError : public std::exception {
 public:
   ResolverOutsideBaseError(std::filesystem::path path,

src/resolver/resolver.cc

@@ -17,6 +17,7 @@
 #include <shared_mutex>  // std::shared_lock
 #include <sstream>       // std::ostringstream
 #include <string>        // std::string
+#include <system_error>  // std::error_code
 #include <unordered_set> // std::unordered_set
 
 static auto
@@ -532,10 +533,18 @@ auto Resolver::add(const std::filesystem::path &collection_relative_path,
 
 auto Resolver::emplace(std::string new_identifier, Entry entry) -> void {
   assert(std::filesystem::exists(entry.path));
-  // As the materialised path must exist if we are emplacing
-  // given a cache hit
   assert(entry.cache_path.has_value());
-  assert(std::filesystem::exists(entry.cache_path.value()));
+  // A cache hit means a finished build wrote this artifact, so it being gone
+  // means the output directory was modified from outside. That violates the
+  // one assumption the cache rests on, and it has to fail the same way in
+  // every build type rather than trip an assertion only where those exist.
+  // The check must not itself throw, since a path the build cannot examine,
+  // whatever the reason, is equally a record it cannot honour
+  std::error_code existence_error;
+  if (!std::filesystem::exists(entry.cache_path.value(), existence_error)) {
+    throw ResolverMissingCachedArtifactError{entry.cache_path.value()};
+  }
+
   assert(entry.collection);
   const auto path{entry.path};
   auto result{this->views.emplace(std::move(new_identifier), std::move(entry))};

test/cli/CMakeLists.txt

@@ -59,6 +59,7 @@ if(ONE_INDEX)
   sourcemeta_one_test_cli(common index fail-option-overflow-value)
   sourcemeta_one_test_cli(common index fail-output-non-directory)
   sourcemeta_one_test_cli(common index fail-page-level-extends)
+  sourcemeta_one_test_cli(common index fail-removed-schema-artifact)
   sourcemeta_one_test_cli(common index fail-resolve-ref-target-foreign-authority)
   sourcemeta_one_test_cli(common index fail-resolve-schema-invalid-uri)
   sourcemeta_one_test_cli(common index fail-resolve-target-chain)

test/cli/index/common/fail-no-arguments.sh

@@ -35,6 +35,14 @@ Advanced Options:
 
      Set the maximum number of direct entries in a directory listing
 
+Output Directory:
+
+   The output directory is owned by the indexer. Do NOT:
+
+   - Manually modify, add, or delete any file inside it
+   - Sync content into it or partially restore it from a backup
+   - Run concurrent indexer invocations over the same output
+
 For more documentation, visit https://one.sourcemeta.com
 EOF
 

test/cli/index/common/fail-removed-schema-artifact.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# The build state records that a build materialised a schema, and an
+# incremental build trusts it. The output directory belongs to the indexer, so
+# a recorded artifact that is gone from disk means something else modified the
+# directory, and the build must refuse loudly instead of continuing over a
+# record it cannot honour
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/one.json"
+{
+  "url": "https://example.com/",
+  "contents": {
+    "schemas": { "baseUri": "https://example.com/", "path": "./schemas" }
+  }
+}
+EOF
+
+mkdir "$TMP/schemas"
+
+cat << 'EOF' > "$TMP/schemas/a.json"
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://example.com/a"
+}
+EOF
+
+"$1" --skip-banner "$TMP/one.json" "$TMP/output" > /dev/null 2>&1
+
+rm "$TMP/output/schemas/schemas/a/%/schema.metapack"
+
+"$1" --skip-banner "$TMP/one.json" "$TMP/output" > "$TMP/output.txt" 2> /dev/null \
+  && CODE="$?" || CODE="$?"
+test "$CODE" = "1" || exit 1
+
+cat << EOF > "$TMP/expected.txt"
+error: The build state references an artifact that no longer exists on disk
+  at path $(realpath "$TMP")/output/schemas/schemas/a/%/schema.metapack
+
+Something other than the indexer modified the output directory, so delete it and index again from scratch
+EOF
+
+diff "$TMP/output.txt" "$TMP/expected.txt"

test/cli/index/common/output-help-skip-banner.sh

@@ -32,6 +32,14 @@ Advanced Options:
 
      Set the maximum number of direct entries in a directory listing
 
+Output Directory:
+
+   The output directory is owned by the indexer. Do NOT:
+
+   - Manually modify, add, or delete any file inside it
+   - Sync content into it or partially restore it from a backup
+   - Run concurrent indexer invocations over the same output
+
 For more documentation, visit https://one.sourcemeta.com
 EOF
 

test/cli/index/common/output-help.sh

@@ -34,6 +34,14 @@ Advanced Options:
 
      Set the maximum number of direct entries in a directory listing
 
+Output Directory:
+
+   The output directory is owned by the indexer. Do NOT:
+
+   - Manually modify, add, or delete any file inside it
+   - Sync content into it or partially restore it from a backup
+   - Run concurrent indexer invocations over the same output
+
 For more documentation, visit https://one.sourcemeta.com
 EOF
 

test/cli/index/community/fail-no-options.sh

@@ -35,6 +35,14 @@ Advanced Options:
 
      Set the maximum number of direct entries in a directory listing
 
+Output Directory:
+
+   The output directory is owned by the indexer. Do NOT:
+
+   - Manually modify, add, or delete any file inside it
+   - Sync content into it or partially restore it from a backup
+   - Run concurrent indexer invocations over the same output
+
 For more documentation, visit https://one.sourcemeta.com
 EOF
 

test/cli/index/community/fail-no-output.sh

@@ -51,6 +51,14 @@ Advanced Options:
 
      Set the maximum number of direct entries in a directory listing
 
+Output Directory:
+
+   The output directory is owned by the indexer. Do NOT:
+
+   - Manually modify, add, or delete any file inside it
+   - Sync content into it or partially restore it from a backup
+   - Run concurrent indexer invocations over the same output
+
 For more documentation, visit https://one.sourcemeta.com
 EOF
 

test/cli/index/enterprise/fail-no-options.sh

@@ -35,6 +35,14 @@ Advanced Options:
 
      Set the maximum number of direct entries in a directory listing
 
+Output Directory:
+
+   The output directory is owned by the indexer. Do NOT:
+
+   - Manually modify, add, or delete any file inside it
+   - Sync content into it or partially restore it from a backup
+   - Run concurrent indexer invocations over the same output
+
 For more documentation, visit https://one.sourcemeta.com
 EOF
 

test/cli/index/enterprise/fail-no-output.sh

@@ -51,6 +51,14 @@ Advanced Options:
 
      Set the maximum number of direct entries in a directory listing
 
+Output Directory:
+
+   The output directory is owned by the indexer. Do NOT:
+
+   - Manually modify, add, or delete any file inside it
+   - Sync content into it or partially restore it from a backup
+   - Run concurrent indexer invocations over the same output
+
 For more documentation, visit https://one.sourcemeta.com
 EOF
 

test/unit/resolver/resolver_test.cc

@@ -694,3 +694,79 @@ TEST(path_url_absolute_ref) {
     "$ref": "2020-12-id"
   })JSON");
 }
+
+TEST(emplace_accepts_a_cache_hit_whose_artifact_is_present) {
+  sourcemeta::one::Resolver resolver{shared_configuration().url};
+  const auto source_path{std::filesystem::path{SCHEMAS_PATH} / "example" /
+                         "2020-12-with-id.json"};
+  resolver.emplace(
+      "http://localhost:8000/example/2020-12-with-id",
+      {.path = source_path,
+       .relative_path = "example/2020-12-with-id",
+       .mtime = std::filesystem::last_write_time(source_path),
+       .evaluate = true,
+       .cache_path = source_path,
+       .dialect = "https://json-schema.org/draft/2020-12/schema",
+       .original_identifier = "https://example.com/schemas/2020-12-with-id",
+       .collection = &std::get<sourcemeta::one::Configuration::Collection>(
+           shared_configuration().entries.at("example"))});
+  EXPECT_EQ(resolver.entry("http://localhost:8000/example/2020-12-with-id")
+                .cache_path.value(),
+            source_path);
+}
+
+TEST(emplace_refuses_a_cache_hit_whose_artifact_cannot_be_examined) {
+  sourcemeta::one::Resolver resolver{shared_configuration().url};
+  const auto source_path{std::filesystem::path{SCHEMAS_PATH} / "example" /
+                         "2020-12-with-id.json"};
+  // A component longer than any filesystem accepts makes examining the path
+  // an error rather than a clean answer, in every environment including
+  // running as root, where permission denials cannot be provoked
+  const auto unexaminable_path{std::filesystem::path{SCHEMAS_PATH} /
+                               std::string(300, 'x') / "schema.metapack"};
+  try {
+    resolver.emplace(
+        "http://localhost:8000/example/2020-12-with-id",
+        {.path = source_path,
+         .relative_path = "example/2020-12-with-id",
+         .mtime = std::filesystem::last_write_time(source_path),
+         .evaluate = true,
+         .cache_path = unexaminable_path,
+         .dialect = "https://json-schema.org/draft/2020-12/schema",
+         .original_identifier = "https://example.com/schemas/2020-12-with-id",
+         .collection = &std::get<sourcemeta::one::Configuration::Collection>(
+             shared_configuration().entries.at("example"))});
+    FAIL();
+  } catch (const sourcemeta::one::ResolverMissingCachedArtifactError &error) {
+    EXPECT_STREQ(
+        error.what(),
+        "The build state references an artifact that no longer exists on disk");
+    EXPECT_EQ(error.path(), unexaminable_path);
+  }
+}
+
+TEST(emplace_refuses_a_cache_hit_whose_artifact_is_gone) {
+  sourcemeta::one::Resolver resolver{shared_configuration().url};
+  const auto source_path{std::filesystem::path{SCHEMAS_PATH} / "example" /
+                         "2020-12-with-id.json"};
+  const std::filesystem::path missing_path{"/no/such/cache/schema.metapack"};
+  try {
+    resolver.emplace(
+        "http://localhost:8000/example/2020-12-with-id",
+        {.path = source_path,
+         .relative_path = "example/2020-12-with-id",
+         .mtime = std::filesystem::last_write_time(source_path),
+         .evaluate = true,
+         .cache_path = missing_path,
+         .dialect = "https://json-schema.org/draft/2020-12/schema",
+         .original_identifier = "https://example.com/schemas/2020-12-with-id",
+         .collection = &std::get<sourcemeta::one::Configuration::Collection>(
+             shared_configuration().entries.at("example"))});
+    FAIL();
+  } catch (const sourcemeta::one::ResolverMissingCachedArtifactError &error) {
+    EXPECT_STREQ(
+        error.what(),
+        "The build state references an artifact that no longer exists on disk");
+    EXPECT_EQ(error.path(), missing_path);
+  }
+}