Enhanced caching to keep track of the commit sha1 for a tag or branch so it knows to invalidate if it is different. This way if someone moves a tag or commits to a branch the cache will know it invalidate and we won't use stale files.

bobgarner · bobgarner · commit 9463d97c9ccf · 2022-06-29T18:13:13.000-07:00
diff --git a/src/main/java/org/entityc/compiler/cmdline/command/CLInfo.java b/src/main/java/org/entityc/compiler/cmdline/command/CLInfo.java
@@ -32,7 +32,7 @@ public CLInfo(CommandLine commandLine) {
 
     @Override
     public void printUsage() {
-        super.printUsageWithArguments("[-s,--sources][-c,--configs][-t,--templates][-g,--generated][-a,--all]");
+        super.printUsageWithArguments("[-s,--sources] [-c,--configs] [-t,--templates] [-g,--generated] [-a,--all]");
     }
 
     @Override
diff --git a/src/main/java/org/entityc/compiler/model/config/MTRepository.java b/src/main/java/org/entityc/compiler/model/config/MTRepository.java
@@ -30,6 +30,7 @@ public class MTRepository extends MTNode {
     private       String              repoName;
     private       boolean             indexedTemplates = false;
     private       String              setupFilename;
+    private       String              commitSHA1;
 
     public MTRepository(ParserRuleContext ctx, String name) {
         super(ctx);
@@ -68,7 +69,7 @@ public MTRepository(String setupUri) {
         Integer lastPathDelimIndex = path.lastIndexOf("/");
         if (lastPathDelimIndex == -1) {
             setupFilename = path;
-            path = "";
+            path          = "";
         } else {
             setupFilename = path.substring(lastPathDelimIndex + 1);
             path          = path.substring(0, lastPathDelimIndex);
@@ -208,4 +209,12 @@ public String getUri() {
         }
         return builder.toString();
     }
+
+    public String getCommitSHA1() {
+        return commitSHA1;
+    }
+
+    public void setCommitSHA1(String commitSHA1) {
+        this.commitSHA1 = commitSHA1;
+    }
 }
diff --git a/src/main/java/org/entityc/compiler/repository/GitHubRepositoryImporter.java b/src/main/java/org/entityc/compiler/repository/GitHubRepositoryImporter.java
@@ -6,12 +6,14 @@
 
 package org.entityc.compiler.repository;
 
+import org.apache.commons.io.IOUtils;
 import org.entityc.compiler.EntityCompiler;
 import org.entityc.compiler.model.config.MTRepository;
 import org.entityc.compiler.model.config.MTRepositoryImport;
 import org.entityc.compiler.util.ECLog;
-import org.apache.commons.io.IOUtils;
+import org.kohsuke.github.GHBranch;
 import org.kohsuke.github.GHContent;
+import org.kohsuke.github.GHRef;
 import org.kohsuke.github.GHRepository;
 import org.kohsuke.github.GitHub;
 
@@ -38,11 +40,11 @@ public RepositoryFile importFromRepository(MTRepository repository, MTRepository
                     "Unable to connect to GitHub: " + repository.getOrganization() + "/" + repository.getRepoName()
                     + "\n" + e.getMessage());
         }
-        String       pathPart    = alternatePath != null ?
-                                   (alternatePath + "/") :
-                                   repository.getPath() != null && !repository.getPath().isEmpty() ?
-                                   repository.getPath() + "/" :
-                                   "";
+        String pathPart = alternatePath != null ?
+                          (alternatePath + "/") :
+                          repository.getPath() != null && !repository.getPath().isEmpty() ?
+                          repository.getPath() + "/" :
+                          "";
         String       gitRepoPath = pathPart + repositoryImport.getFilename() + "." + extension;
         GHRepository repo        = null;
         try {
@@ -61,6 +63,7 @@ public RepositoryFile importFromRepository(MTRepository repository, MTRepository
                                "Could not find file in repository: " + cachedRepositoryFile.getFilepath() + " with tag "
                                + repository.getTag());
             }
+
             String outputFilepath = cachedRepositoryFile.getFilepath();
             // in case the filename has sub directories, make sure they are created
             if (outputFilepath.contains(File.separator)) {
@@ -90,6 +93,50 @@ public RepositoryFile importFromRepository(MTRepository repository, MTRepository
         return cachedRepositoryFile;
     }
 
+    @Override
+    public void updateRepositoryCommitSHA1(MTRepository repository) {
+
+        if (repository.getCommitSHA1() != null) {
+            return; // we are good
+        }
+
+        try {
+            if (github == null) {
+                github = GitHub.connect();
+            }
+        } catch (IOException e) {
+            ECLog.logFatal(
+                    "Unable to connect to GitHub: " + repository.getOrganization() + "/" + repository.getRepoName()
+                    + "\n" + e.getMessage());
+        }
+
+        GHRepository repo = null;
+        try {
+            repo = github.getRepository(repository.getOrganization() + "/" + repository.getRepoName());
+        } catch (IOException e) {
+            ECLog.logFatal("Unable to connect to repository: " + repository.getOrganization() + "/"
+                           + repository.getRepoName());
+        }
+
+        String sha1 = null;
+        try {
+            GHBranch branch = repo.getBranch(repository.getTag());
+            if (branch != null) {
+                sha1 = branch.getSHA1();
+            }
+        } catch (IOException e) {
+            // could not find the commit, maybe its not a branch name
+        }
+        try {
+            GHRef ref = repo.getRef("tags/" + repository.getTag());
+            sha1 = ref.getObject().getSha();
+        } catch (IOException e) {
+            // must not be a tag
+        }
+
+        repository.setCommitSHA1(sha1);
+    }
+
     @Override
     public void close() {
         github = null;
diff --git a/src/main/java/org/entityc/compiler/repository/LocalRepositoryImporter.java b/src/main/java/org/entityc/compiler/repository/LocalRepositoryImporter.java
@@ -52,6 +52,11 @@ public RepositoryFile importFromRepository(MTRepository repository, MTRepository
         return cachedRepositoryFile;
     }
 
+    @Override
+    public void updateRepositoryCommitSHA1(MTRepository repository) {
+        // nothing to do for this type of repository
+    }
+
     @Override
     public void close() {
     }
diff --git a/src/main/java/org/entityc/compiler/repository/RepositoryCache.java b/src/main/java/org/entityc/compiler/repository/RepositoryCache.java
@@ -6,10 +6,13 @@
 
 package org.entityc.compiler.repository;
 
+import org.apache.commons.io.FileUtils;
 import org.entityc.compiler.model.config.MTRepository;
 import org.entityc.compiler.model.config.MTRepositoryType;
+import org.entityc.compiler.util.ECLog;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.UUID;
 
 public class RepositoryCache {
@@ -32,10 +35,6 @@ public RepositoryCache(CacheStructure structure) {
         baseCacheDirectory.mkdirs();
     }
 
-    public File getBaseCacheDirectory() {
-        return baseCacheDirectory;
-    }
-
     public RepositoryFile getRepositoryFile(MTRepository repository, String filename, boolean asInclude) {
         String outputFilepath = baseCacheDirectory.getAbsolutePath() + File.separator;
 
@@ -44,23 +43,71 @@ public RepositoryFile getRepositoryFile(MTRepository repository, String filename
                 outputFilepath += filename;
                 break;
             case UserCache: {
-                String pathPart = repository.getPath() != null && !repository.getPath().isEmpty() ? repository.getPath() + File.separator : "";
+                String pathPart         = repository.getPath() != null && !repository.getPath().isEmpty() ?
+                                          repository.getPath() + File.separator :
+                                          "";
                 String completeFilename = pathPart + filename;
                 if (repository.getType() == MTRepositoryType.LOCAL) {
                     outputFilepath = completeFilename;
                 } else {
                     outputFilepath += getRepositoryCachePath(repository) + File.separator + completeFilename;
                 }
             }
-                break;
+            break;
         }
 
         RepositoryFile repositoryFile = new RepositoryFile(repository, outputFilepath, asInclude);
         return repositoryFile;
     }
 
     private String getRepositoryCachePath(MTRepository repository) {
-        return repository.getOrganization() + File.separator + repository.getRepoName() + File.separator + repository.getTag();
+        return repository.getOrganization() + File.separator + repository.getRepoName() + File.separator
+               + repository.getTag();
+    }
+
+    public void validateRepositoryInCache(MTRepository repository) {
+        if (structure != CacheStructure.UserCache || repository.getType() == MTRepositoryType.LOCAL || !repository.isValid()) {
+            return; // not this one
+        }
+        String commitSha1Filename = getBaseCacheDirectory().getAbsolutePath() + File.separator + getRepositoryCachePath(
+                repository) + File.separator + ".commitSha1";
+        File   file               = new File(commitSha1Filename);
+        String cacheSHA1          = null;
+        if (file.exists()) {
+            try {
+                cacheSHA1 = FileUtils.readFileToString(file);
+            } catch (IOException e) {
+                // oh well, assume it is not there
+            }
+        }
+        String repositoryCommitSHA1 = repository.getCommitSHA1();
+        if (repositoryCommitSHA1 == null || cacheSHA1 == null || !repository.getCommitSHA1().equals(cacheSHA1)) {
+            invalidateCacheDirectory(repository);
+            if (repositoryCommitSHA1 != null) {
+                try {
+                    FileUtils.write(file, repositoryCommitSHA1);
+                } catch (IOException e) {
+                    ECLog.logWarning("Unable to record repository sha1 in cache directory: " + commitSha1Filename);
+                }
+            }
+        }
+    }
+
+    public void invalidateCacheDirectory(MTRepository repository) {
+        if (structure != CacheStructure.UserCache) {
+            return;
+        }
+        String fullPath = getBaseCacheDirectory().getAbsolutePath() + File.separator + getRepositoryCachePath(
+                repository);
+        try {
+            FileUtils.cleanDirectory(new File(fullPath));
+        } catch (IOException e) {
+            ECLog.logWarning("Unable to remove cache directory: " + fullPath);
+        }
+    }
+
+    public File getBaseCacheDirectory() {
+        return baseCacheDirectory;
     }
 
     public enum CacheStructure {
diff --git a/src/main/java/org/entityc/compiler/repository/RepositoryImportManager.java b/src/main/java/org/entityc/compiler/repository/RepositoryImportManager.java
@@ -47,6 +47,11 @@ public RepositoryFile getRepositoryFileByName(String identifier) {
         return repositoryFilesByIdentifier.get(identifier);
     }
 
+    public void updateRepositoryCommitSHA1(MTRepository repository) {
+        RepositoryImporter repositoryImporter = importersByType.get(repository.getType());
+        repositoryImporter.updateRepositoryCommitSHA1(repository);
+    }
+
     public RepositoryFile importFromRepository(MTSpace space, MTRepositoryImport repositoryImport,
                                                String extension, boolean asInclude) {
 
@@ -67,15 +72,20 @@ public RepositoryFile importFromRepository(MTSpace space, MTRepositoryImport rep
                                            && type == MTRepositoryType.LOCAL;
 
         MTRepository sourceRepository = repository;
-        String alternatePath = null;
+        String       alternatePath    = null;
         if (shouldUseSpaceRepository) {
             sourceRepository = space.getRepositoryThatImportedThisSpace();
-            alternatePath = repository.getPath();
+            alternatePath    = repository.getPath();
         }
 
         RepositoryImporter repositoryImporter = importersByType.get(sourceRepository.getType());
-        RepositoryFile cacheRepositoryFile = repositoryCache.getRepositoryFile(sourceRepository, filename, asInclude);
-        if (sourceRepository.getType() != MTRepositoryType.LOCAL && cacheRepositoryFile.exists()) { // check if it exists in the cache already
+        repositoryImporter.updateRepositoryCommitSHA1(
+                sourceRepository); // if it has one, fetch it so the cache will know if it needs to invalidate itself
+        repositoryCache.validateRepositoryInCache(sourceRepository);
+        RepositoryFile cacheRepositoryFile = repositoryCache.getRepositoryFile(sourceRepository, filename,
+                                                                               asInclude);
+        if (sourceRepository.getType() != MTRepositoryType.LOCAL
+            && cacheRepositoryFile.exists()) { // check if it exists in the cache already
             if (EntityCompiler.isVerbose()) {
                 ECLog.logInfo("Found file already in cache: " + cacheRepositoryFile.getFilepath());
             }
@@ -84,7 +94,7 @@ public RepositoryFile importFromRepository(MTSpace space, MTRepositoryImport rep
         }
 
         RepositoryFile file = repositoryImporter.importFromRepository(sourceRepository, repositoryImport,
-                                                                             cacheRepositoryFile, extension, alternatePath);
+                                                                      cacheRepositoryFile, extension, alternatePath);
         if (file == null || !file.exists()) {
             String nameOrPath = repository.getType() == MTRepositoryType.LOCAL ?
                                 repository.getPath() :
diff --git a/src/main/java/org/entityc/compiler/repository/RepositoryImporter.java b/src/main/java/org/entityc/compiler/repository/RepositoryImporter.java
@@ -12,6 +12,7 @@
 public interface RepositoryImporter {
 
     RepositoryFile importFromRepository(MTRepository repository, MTRepositoryImport repositoryImport, RepositoryFile cachedRepositoryFile, String extension, String alternatePath);
+    void updateRepositoryCommitSHA1(MTRepository repository);
 
     void close();
 }

Original file line number	Diff line number	Diff line change
`@@ -32,7 +32,7 @@ public CLInfo(CommandLine commandLine) {`
`32`	`32`
`33`	`33`	`@Override`
`34`	`34`	`public void printUsage() {`
`35`		`- super.printUsageWithArguments("[-s,--sources][-c,--configs][-t,--templates][-g,--generated][-a,--all]");`
	`35`	`+ super.printUsageWithArguments("[-s,--sources] [-c,--configs] [-t,--templates] [-g,--generated] [-a,--all]");`
`36`	`36`	`}`
`37`	`37`
`38`	`38`	`@Override`
Original file line number	Diff line number	Diff line change
`@@ -52,6 +52,11 @@ public RepositoryFile importFromRepository(MTRepository repository, MTRepository`
`52`	`52`	`return cachedRepositoryFile;`
`53`	`53`	`}`
`54`	`54`
	`55`	`+ @Override`
	`56`	`+ public void updateRepositoryCommitSHA1(MTRepository repository) {`
	`57`	`+ // nothing to do for this type of repository`
	`58`	`+ }`
	`59`	`+`
`55`	`60`	`@Override`
`56`	`61`	`public void close() {`
`57`	`62`	`}`
Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,7 @@`
`12`	`12`	`public interface RepositoryImporter {`
`13`	`13`
`14`	`14`	`RepositoryFile importFromRepository(MTRepository repository, MTRepositoryImport repositoryImport, RepositoryFile cachedRepositoryFile, String extension, String alternatePath);`
	`15`	`+ void updateRepositoryCommitSHA1(MTRepository repository);`
`15`	`16`
`16`	`17`	`void close();`
`17`	`18`	`}`