Enforce development environment instructions to Elasticsearch 8.11.3 (geonetwork#7866)

jodygarnett · josegar74 · web-flow · commit e7d520bab030 · 2024-04-15T15:15:18.000+02:00
* docker - missing mentioned kibana configuration - missing kibana configuration provided by Pierre Mauduit (via geonetwork#6284) - using latest geonetwork image (rather htan 3.99.0) - Setup and document docker folder as independent testing environments * reorder es/README.md to list recommended best practice first * Use enforcer plugin to check es/README.md and es/docker-compose.yml * Also enforce version check of es-dashboard/README.md * Additional troubleshooting advise for low disk space and blocked index read-only * Apply suggestions from code review Accept feedback from review Co-authored-by: Jose García <josegar74@gmail.com> --------- Co-authored-by: Jose García <josegar74@gmail.com>
diff --git a/docker/README.md b/docker/README.md
@@ -0,0 +1,3 @@
+# Docker Test Environments
+
+These docker images are intended for development and debugging. For production we recommend the official GeoNetwork docker images at https://github.com/geonetwork/docker-geonetwork.git repository.
diff --git a/docker/docker-geonetwork.txt b/docker/docker-geonetwork.txt
diff --git a/docker/gn-cas-ldap/README.md b/docker/gn-cas-ldap/README.md
@@ -1,11 +1,13 @@
-# Introduction
+# GeoNetwork CAS Test Environment
 
 This composition is meant to make runtime testing the CAS integration of
 GeoNetwork easier.
 
 This composition also integrates a LDAP, so that testing the
 config-spring-cas-ldap configuration is also possible.
 
+These docker images are intended for development and debugging. For production we recommend the official GeoNetwork docker images at https://github.com/geonetwork/docker-geonetwork.git repository.
+
 # Prerequisites
 
 It requires the GeoNetwork webapp to be built first:
@@ -18,7 +20,6 @@ Then it can be launched:
 
 ```
 $ docker-compose up
-
 ```
 
 # Accessing the CAS login page from GeoNetwork
diff --git a/docker/gn-postgres/README.md b/docker/gn-postgres/README.md
@@ -0,0 +1,8 @@
+# GeoNetwork PostgreSQL Test Environment
+
+This composition is meant to make runtime testing the PostgreSQL integration of
+GeoNetwork easier.
+
+This folder provides a ``docker-compose.yml`` file for local testing.
+
+These docker images are intended for development and debugging. For production we recommend the official GeoNetwork docker images at https://github.com/geonetwork/docker-geonetwork.git repository.
diff --git a/docker/gn-postgres/docker-compose.yml b/docker/gn-postgres/docker-compose.yml
@@ -4,7 +4,7 @@ volumes:
 
 services:
   geonetwork:
-    image: geonetwork:3.99.0
+    image: geonetwork:latest
     restart: always
     ports:
       - 8080:8080
diff --git a/docker/gn-postgres/kibana/kibana.txt b/docker/gn-postgres/kibana/kibana.txt
@@ -0,0 +1,5 @@
+server.basePath: "/geonetwork/dashboards"
+server.rewriteBasePath: false
+kibana.index: ".dashboards"
+elasticsearch.hosts: ["http://elasticsearch:9200"]
+
diff --git a/es/README.md b/es/README.md
@@ -1,33 +1,48 @@
 # Install, configure and start Elasticsearch
 
+## Installation options
+
 This section describes several methods for configuring Elasticsearch for development.
 
 These configurations should not be used for a production deployment.
 
-## Manual installation
+### Docker installation (Recommended)
 
-1. Download Elasticsearch 8.x (tested with 8.11.3 for Geonetwork 4.4.x) from https://www.elastic.co/downloads/elasticsearch
-and copy to the ES module, e.g., es/elasticsearch-8.11.3
+1. Use docker pull to download the image (you can check version in the :file:`pom.xml` file):
 
-2. Disable the security
+   ```
+   docker pull docker.elastic.co/elasticsearch/elasticsearch:8.11.3
+   ```
 
-Elasticsearch 8 has security enabled by default. To disable this configuration for development, update the file `config/elasticsearch.yml` adding at the end:
+2. Use docker run, leaving 9200 available:
 
-```
-xpack.security.enabled: false
-xpack.security.enrollment.enabled: false
-```
+   ```
+   docker run -p 9200:9200 -p 9300:9300 \ 
+      -e "discovery.type=single-node" \
+      -e "xpack.security.enabled=false" \
+      -e "xpack.security.enrollment.enabled=false" \ 
+      docker.elastic.co/elasticsearch/elasticsearch:8.11.3
+   ```
+
+3. Check that elasticsearch is running by visiting http://localhost:9200 in a browser
 
+### Docker compose installation
 
-3. Start ES using:
+1. Use docker compose with the provided [docker-compose.yml](docker-compose.yml):
 
-   ```shell script
-   ./bin/elasticsearch
+   ```
+   cd es
+   docker-compose up
    ```
 
-4. Check that elasticsearch is running by visiting http://localhost:9200 in a browser
+3. Check that it is running using your browser:
+   
+   * Elasticsearch: http://localhost:9200
+   * Kibana: http://localhost:5601
+
+### Maven installation
 
-## Maven installation
+Maven installation ensure you always are using the ``es.version`` version specified in ``pom.xml``.
 
 1. Maven can take care of the installation steps:
 
@@ -44,42 +59,32 @@ xpack.security.enrollment.enabled: false
    ```
 3. Check that elasticsearch is running by visiting http://localhost:9200 in a browser
 
-## Docker installation
+## Manual installation
 
-1. Use docker pull to download the image (you can check version in the :file:`pom.xml` file):
+1. Download Elasticsearch 8.11.3 from https://www.elastic.co/downloads/elasticsearch
+and copy to the ES module, e.g., ``es/elasticsearch-8.11.3`
 
-   ```
-   docker pull docker.elastic.co/elasticsearch/elasticsearch:8.11.3
-   ```
+2. Disable the security
 
-2. Use docker run, leaving 9200 available:
+   Elasticsearch 8 has security enabled by default. To disable this configuration for development, update the file `config/elasticsearch.yml` adding at the end:
 
    ```
-   docker run -p 9200:9200 -p 9300:9300 \ 
-      -e "discovery.type=single-node" \
-      -e "xpack.security.enabled=false" \
-      -e "xpack.security.enrollment.enabled=false" \ 
-      docker.elastic.co/elasticsearch/elasticsearch:8.11.3
+   xpack.security.enabled: false
+   xpack.security.enrollment.enabled: false
    ```
 
-3. Check that elasticsearch is running by visiting http://localhost:9200 in a browser
-
-## Docker compose installation
-
-1. Use docker compose with the provided [docker-compose.yml](docker-compose.yml):
+3. Start ES using:
 
-   ```
-   cd es
-   docker-compose up
+   ```shell script
+   ./bin/elasticsearch
    ```
 
-3. Check that it is running using your browser:
-   
-   * Elasticsearch: http://localhost:9200
-   * Kibana: http://localhost:5601
+4. Check that elasticsearch is running by visiting http://localhost:9200 in a browser
 
 # Configuration
 
+## Index management
+
 Optionally you can manually create index but they will be created by the catalogue when 
 the Elastic instance is available and if index does not exist.
 
@@ -122,7 +127,7 @@ Don't hesitate to propose a Pull Request with the new language.
 
 1. Configure ES to start on server startup. It is recommended to protect `gn-records` index from the Internet access.
 
-   * Note that for debian-based servers the current deb download (7.3.2) can be installed rather than installing manually and can be configured to run as a service using the instructions here: https://www.elastic.co/guide/en/elasticsearch/reference/current/starting-elasticsearch.html
+   * Note that for debian-based servers the current deb download (8.11.3) can be installed rather than installing manually and can be configured to run as a service using the instructions here: https://www.elastic.co/guide/en/elasticsearch/reference/current/starting-elasticsearch.html
 
 
 # Troubleshoot
@@ -163,3 +168,22 @@ field expansion for [*] matches too many fields, limit: 1024
 An option is to restrict `queryBase` to limit the number of field to query on. `any:(${any}) resourceTitleObject.default:(${any})^2` is a good default. Using `${any}` will probably trigger the error if the number of records is high.
 
 The other option is to increase `indices.query.bool.max_clause_count`.
+
+
+## Disk space threshold
+
+The server application will refuse to write new content unless there is enough free space available (by default 1/4 of your hard drive).
+
+To turn off this check:
+
+```
+ curl -XPUT http://localhost:9200/_cluster/settings -H 'Content-Type: application/json' -d '{ "transient" : { "cluster.routing.allocation.disk.threshold_enabled" : false } }' 
+```
+
+## Blocked by index read-only / allow delete
+
+To recover:
+
+```
+curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}'
+```
diff --git a/es/es-dashboards/README.md b/es/es-dashboards/README.md
@@ -1,24 +1,20 @@
 # Install, configure and start Kibana
 
-## Manual installation
-
-Download Kibana from https://www.elastic.co/downloads/kibana. For Geonetwork 3.8.x download at least version 7.2.1
+## Installation options
 
-Set Kibana base path and index name in config/kibana.yml:
+### Docker compose installation (Recommended)
 
-```
-server.basePath: "/geonetwork/dashboards"
-server.rewriteBasePath: false
-```
+1. Use docker compose with the provided [docker-compose.yml](es/docker-compose.yml):
 
-Adapt if needed ```elasticsearch.url``` and ```server.host```.
-
-Start Kibana manually:
+   ```
+   cd es
+   docker-compose up
+   ```
 
-```
-cd kibana/bin
-./kibana
-```
+3. Check that it is running using your browser:
+   
+   * Elasticsearch: http://localhost:9200
+   * Kabana: http://localhost:5601
 
 ## Maven installation
 
@@ -41,20 +37,26 @@ cd kibana/bin
    mvn exec:exec -Dkb-start
    ```
 
-## Docker compose installation
+## Manual installation
+
+1. Download Kibana 8.11.3 from https://www.elastic.co/downloads/kibana
+
+2. Set Kibana base path and index name in config/kibana.yml:
+
+   ```
+   server.basePath: "/geonetwork/dashboards"
+   server.rewriteBasePath: false
+   ```
+
+3. Adapt if needed ```elasticsearch.url``` and ```server.host```.
 
-1. Use docer compose with the provided [docker-compose.yml](docker-compose.yml):
+4. Start Kibana manually:
 
    ```
-   cd es
-   docker-compose up
+   cd kibana/bin
+   ./kibana
    ```
 
-3. Check that it is running using your browser:
-   
-   * Elasticsearch: http://localhost:9200
-   * Kabana: http://localhost:5601
-   
 ## Import Configuration
 
 1. Kibana should be running from:
@@ -69,16 +71,17 @@ cd kibana/bin
    http://localhost:8080/geonetwork/dashboards
    ```
 
+
 ## Troubleshoot
 
 If it does not start properly, check Kibana log files (eg. it may fail if Elasticsearch version
 is not compatible with Kibana version).
 
-Visit Kibana in a browser using one of the above links and go to 'Saved Objects'. Import export.ndjson from https://github.com/geonetwork/core-geonetwork/blob/4.0.x/es/es-dashboards/data/export.ndjson
+Visit Kibana in a browser using one of the above links and go to 'Saved Objects'. Import export.ndjson from https://github.com/geonetwork/core-geonetwork/blob/main/es/es-dashboards/data/export.ndjson
 
 ### Production Use
 
-Kibana can be installed from the debian files, and 7.3.2 is confirmed as working with Geonetwork 3.8.x.
+Kibana can be installed from the debian files, and Kibana 8.11.3 is confirmed as working with Geonetwork 4.4.x.
 
 Set Kibana to start when the server starts up, using the instructions at https://www.elastic.co/guide/en/kibana/current/start-stop.html
 
diff --git a/es/es-dashboards/pom.xml b/es/es-dashboards/pom.xml
@@ -24,12 +24,74 @@
   <modelVersion>4.0.0</modelVersion>
   <artifactId>gn-es-dashboards</artifactId>
   <name>GeoNetwork dashboard app based on Kibana</name>
-
   <parent>
     <artifactId>gn-es</artifactId>
     <groupId>org.geonetwork-opensource</groupId>
     <version>4.4.4-SNAPSHOT</version>
   </parent>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-enforcer-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>check-readme</id>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <configuration combine.self="override">
+              <rules>
+                <evaluateBeanshell>
+                  <message>Update README.md examples for Elasticsearch ${es.version}</message>
+                  <condition>
+                      import java.util.regex.Pattern;
+
+                      esVersion = "${es.version}";
+                      print("Scanning README for " + esVersion);
+
+                      docker = Pattern.compile("Kibana (\\d.\\d\\d.\\d)");
+                      download = Pattern.compile("Download Kibana (\\d.\\d\\d.\\d)");
+
+                      patterns = new Pattern[]{ docker, download};
+
+                      readme = new BufferedReader(new FileReader("README.md"));
+
+                      number = 0;
+                      while ((line = readme.readLine()) != null) {
+                        number++;
+                        for (pattern : patterns ){
+                          matcher = pattern.matcher(line);
+                          if (matcher.find()) {
+                            if (!esVersion.equals(matcher.group(1))) {
+                              print("README.md:"+number+" FAILURE: " + line);
+                              return false;
+                            }
+                          }
+                        }
+                      }
+                      readme.close();
+                    true;
+                  </condition>
+                </evaluateBeanshell>
+              </rules>
+            </configuration>
+          </execution>
+          <execution>
+            <id>check-docker</id>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <configuration combine.self="override">
+              <skip>true</skip>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
   <profiles>
     <profile>
       <id>kb-download</id>
diff --git a/es/pom.xml b/es/pom.xml

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+# Docker Test Environments`
	`2`	`+`
	`3`	`+These docker images are intended for development and debugging. For production we recommend the official GeoNetwork docker images at https://github.com/geonetwork/docker-geonetwork.git repository.`