Fix submodules path in all files

Author: cgr71ii

Diff for: .github/workflows/build-and-test.yaml

@@ -54,14 +54,14 @@ jobs:
       -  name: Install additional python requirements
          run: |
             pip3 install .[all]
-            pip3 install ./bicleaner
-            pip3 install ./bicleaner-ai
-            pip3 install ./kenlm --install-option="--max_order 7"
-            pip3 install ./bifixer
-            pip3 install ./biroamer
+            pip3 install ./third_party/bicleaner
+            pip3 install ./third_party/bicleaner-ai
+            pip3 install ./third_party/kenlm --install-option="--max_order 7"
+            pip3 install ./third_party/bifixer
+            pip3 install ./third_party/biroamer
             python3 -c "from flair.models import SequenceTagger; SequenceTagger.load('flair/ner-english-fast')"
-            pip3 install ./neural-document-aligner
-            pip3 install ./vecalign
+            pip3 install ./third_party/neural-document-aligner
+            pip3 install ./third_party/vecalign
       -  name: Setup golang
          uses: actions/setup-go@v2
          with:

Diff for: .github/workflows/deferred.yaml

@@ -54,12 +54,12 @@ jobs:
       -  name: Install additional python requirements
          run: |
             pip3 install .[all]
-            pip3 install ./bifixer
-            pip3 install ./bicleaner
-            pip3 install ./kenlm --install-option="--max_order 7"
-            pip3 install ./biroamer
+            pip3 install ./third_party/bifixer
+            pip3 install ./third_party/bicleaner
+            pip3 install ./third_party/kenlm --install-option="--max_order 7"
+            pip3 install ./third_party/biroamer
             python3 -c "from flair.models import SequenceTagger; SequenceTagger.load('flair/ner-english-fast')"
-            pip3 install -r deferred-crawling/requirements.txt
+            pip3 install -r ./third_party/deferred-crawling/requirements.txt
       -  name: Setup golang
          uses: actions/setup-go@v2
          with:

Diff for: CMakeLists.txt

@@ -4,7 +4,7 @@ set(CMAKE_CXX_STANDARD 11)
 
 project(bitextor)
 
-set(PREPROCESS_PATH "${PROJECT_SOURCE_DIR}/preprocess")
+set(PREPROCESS_PATH "${PROJECT_SOURCE_DIR}/third_party/preprocess")
 set(SKIP_PREPROCESS_BUILD ON) # avoid compiling preprocess a million times
 set(CMAKE_BUILD_TYPE "Release")
 include(GNUInstallDirs)
@@ -15,26 +15,26 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
 
 if(NOT SKIP_PREPROCESS)
-    add_subdirectory(${PROJECT_SOURCE_DIR}/preprocess preprocess)
+    add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/preprocess preprocess)
     install(TARGETS b64filter cache docenc foldfilter mmhsum
         DESTINATION ${CMAKE_INSTALL_BINDIR}
     )
 endif()
 
 if(NOT SKIP_WARC2TEXT)
-    add_subdirectory(${PROJECT_SOURCE_DIR}/warc2text warc2text)
+    add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/warc2text warc2text)
 endif()
 
 if(NOT SKIP_PREVERTICAL2TEXT)
-    add_subdirectory(${PROJECT_SOURCE_DIR}/prevertical2text prevertical2text)
+    add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/prevertical2text prevertical2text)
 endif()
 
 if(NOT SKIP_BLEUALIGN)
-    add_subdirectory(${PROJECT_SOURCE_DIR}/bleualign-cpp bleualign-cpp)
+    add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/bleualign-cpp bleualign-cpp)
 endif()
 
 if (NOT SKIP_HUNALIGN)
-    add_subdirectory(${PROJECT_SOURCE_DIR}/hunalign/src)
+    add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/hunalign/src)
     install(PROGRAMS ${PROJECT_BINARY_DIR}/bin/hunalign
         DESTINATION ${CMAKE_INSTALL_BINDIR}
     )
@@ -46,14 +46,14 @@ if (NOT SKIP_DOCALIGN)
 endif()
 
 if (NOT SKIP_BIROAMER)
-    add_subdirectory(${PROJECT_SOURCE_DIR}/biroamer fast_align)
+    add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/biroamer fast_align)
 endif()
 
 if (NOT SKIP_MGIZA)
-    add_subdirectory(${PROJECT_SOURCE_DIR}/mgiza/mgizapp mgizapp)
+    add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/mgiza/mgizapp mgizapp)
 endif()
 
 if (NOT SKIP_KENLM)
     set(KENLM_MAX_ORDER 7)
-    add_subdirectory(${PROJECT_SOURCE_DIR}/kenlm kenlm)
+    add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/kenlm kenlm)
 endif()

Diff for: Dockerfile

@@ -83,13 +83,13 @@ RUN pip3 install --upgrade pip
 ## bitextor
 RUN pip3 install .[all]
 ## bicleaner
-RUN pip3 install ./bicleaner
-RUN pip3 install ./bicleaner-ai
-RUN pip3 install ./kenlm --install-option="--max_order=7"
+RUN pip3 install ./third_party/bicleaner
+RUN pip3 install ./third_party/bicleaner-ai
+RUN pip3 install ./third_party/kenlm --install-option="--max_order=7"
 ##  bifixer
-RUN pip3 install ./bifixer
+RUN pip3 install ./third_party/bifixer
 ## biroamer
-RUN pip3 install ./biroamer
+RUN pip3 install ./third_party/biroamer
 RUN python3 -c "from flair.models import SequenceTagger; SequenceTagger.load('flair/ner-english-fast')"
 ## cld3
 RUN pip3 install Cython

Diff for: conda-build/build.sh

@@ -27,17 +27,17 @@ $PYTHON -m pip install .[all]
 ### FastSpell (bicleaner-hardrules)
 INCLUDE_PATH="$PREFIX/include" $PYTHON -m pip install hunspell
 ### Bicleaner, Bicleaner AI and KenLM
-$PYTHON -m pip install ./bicleaner
-$PYTHON -m pip install ./bicleaner-ai
-$PYTHON -m pip install ./kenlm --install-option="--max_order 7"
+$PYTHON -m pip install ./third_party/bicleaner
+$PYTHON -m pip install ./third_party/bicleaner-ai
+$PYTHON -m pip install ./third_party/kenlm --install-option="--max_order 7"
 ### Bifixer
-$PYTHON -m pip install ./bifixer
+$PYTHON -m pip install ./third_party/bifixer
 ### Biroamer and model
-$PYTHON -m pip install ./biroamer && \
+$PYTHON -m pip install ./third_party/biroamer && \
 $PYTHON -c "from flair.models import SequenceTagger; SequenceTagger.load('flair/ner-english-fast')"
 ### Neural Document Aligner
-$PYTHON -m pip install ./neural-document-aligner
-$PYTHON -m pip install ./vecalign
+$PYTHON -m pip install ./third_party/neural-document-aligner
+$PYTHON -m pip install ./third_party/vecalign
 ## CLD3
 $PYTHON -m pip install Cython
 $PYTHON -m pip install pycld3

Diff for: docs/INSTALL.md

@@ -169,15 +169,15 @@ pip3 install --upgrade pip
 # bitextor (mandatory dependencies):
 pip3 install .
 # additional dependencies:
-pip3 install ./bifixer
-pip3 install ./bicleaner
-pip3 install ./bicleaner-ai
-pip3 install ./kenlm --install-option="--max_order 7" # Bicleaner and Bicleaner AI dependency
-pip3 install ./biroamer && \
+pip3 install ./third_party/bifixer
+pip3 install ./third_party/bicleaner
+pip3 install ./third_party/bicleaner-ai
+pip3 install ./third_party/kenlm --install-option="--max_order 7" # Bicleaner and Bicleaner AI dependency
+pip3 install ./third_party/biroamer && \
 python3 -c "from flair.models import SequenceTagger; SequenceTagger.load('flair/ner-english-fast')"
 ## neural
-pip3 install ./neural-document-aligner
-pip3 install ./vecalign
+pip3 install ./third_party/neural-document-aligner
+pip3 install ./third_party/vecalign
 ```
 
 If you don't want to install all Python requirements in `requirements.txt` because you don't expect to run some of Bitextor modules, you can comment those `*.txt` in `requirements.txt` and rerun Bitextor installation. Also, there are different optional packages that you can choose depending on your needs instead of install all of them:

Diff for: setup.py

@@ -52,7 +52,7 @@ def get_extras_require():
     opt_requirements = get_extras_require()
 
     # Copy necessary files so they will be available when the package is built
-    copytree("preprocess/moses", os.path.join(wd, "bitextor/data/moses"))
+    copytree("third_party/preprocess/moses", os.path.join(wd, "bitextor/data/moses"))
 
     setuptools.setup(
         name="bitextor",