diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fb0c8e98cf..a89b30828d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,6 +58,7 @@ jobs: fail-fast: false matrix: command: [ "server-app:runContextRecreatingTests", "server-app:runStandardTests", "data:test" ] + profile: [ "default", "cockroach" ] steps: - uses: actions/checkout@v2 @@ -91,7 +92,7 @@ jobs: tar -xzf ~/backend-testing.tgz ./backend/testing/build - name: Run backend tests - run: ./gradlew ${{ matrix.command }} + run: SPRING_PROFILES_ACTIVE=${{ matrix.profile }} ./gradlew ${{ matrix.command }} env: SKIP_SERVER_BUILD: true @@ -104,7 +105,7 @@ jobs: - uses: actions/upload-artifact@v2 if: always() with: - name: backend_test_reports_${{ steps.version.outputs.reportName }} + name: backend_test_reports_${{ matrix.profile }} path: | ./**/build/reports/**/* @@ -117,6 +118,7 @@ jobs: matrix: total_jobs: [ 10 ] job_index: [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] + profile: ["cockroach", "default"] steps: - uses: actions/checkout@v2 @@ -187,6 +189,7 @@ jobs: SKIP_INSTALL_E2E_DEPS: true E2E_TOTAL_JOBS: ${{matrix.total_jobs}} E2E_JOB_INDEX: ${{matrix.job_index}} + SPRING_PROFILES_ACTIVE: ${{matrix.profile}} - uses: actions/upload-artifact@v2 if: failure() diff --git a/backend/app/build.gradle b/backend/app/build.gradle index b5ffe3b626..441e4c0922 100644 --- a/backend/app/build.gradle +++ b/backend/app/build.gradle @@ -167,19 +167,22 @@ dependencies { test { useJUnitPlatform() maxHeapSize = "2048m" - //maxParallelForks = (int) (Runtime.runtime.availableProcessors() / 2 + 1) } task runContextRecreatingTests(type: Test, group: 'verification') { + dependsOn "startCockroachDb" useJUnitPlatform { includeTags "contextRecreating" } + finalizedBy "stopCockroachDb" } task runStandardTests(type: Test, group: 'verification') { + dependsOn "startCockroachDb" useJUnitPlatform { excludeTags "contextRecreating" } + finalizedBy "stopCockroachDb" } springBoot { diff --git a/backend/app/src/main/kotlin/io/tolgee/Application.kt b/backend/app/src/main/kotlin/io/tolgee/Application.kt index 53f90bbeb3..a8e8cd3013 100644 --- a/backend/app/src/main/kotlin/io/tolgee/Application.kt +++ b/backend/app/src/main/kotlin/io/tolgee/Application.kt @@ -20,6 +20,7 @@ import org.springframework.boot.autoconfigure.ldap.LdapAutoConfiguration import org.springframework.boot.context.properties.ConfigurationPropertiesScan import org.springframework.data.jpa.repository.config.EnableJpaAuditing import org.springframework.data.jpa.repository.config.EnableJpaRepositories +import org.springframework.retry.annotation.EnableRetry @SpringBootApplication( exclude = [ @@ -43,6 +44,7 @@ import org.springframework.data.jpa.repository.config.EnableJpaRepositories @EntityScan("io.tolgee.model") @ConfigurationPropertiesScan @EnableJpaRepositories("io.tolgee.repository") +@EnableRetry class Application { companion object { @JvmStatic diff --git a/backend/app/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt b/backend/app/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt index e0b8165893..50eddaeade 100644 --- a/backend/app/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt +++ b/backend/app/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt @@ -350,7 +350,7 @@ class V2ImportController( } private fun checkLanguageFromProject(languageId: Long): Language { - val existingLanguage = languageService.findById(languageId).orElse(null) ?: throw NotFoundException() + val existingLanguage = languageService.find(languageId) ?: throw NotFoundException() if (existingLanguage.project.id != projectHolder.project.id) { throw BadRequestException(io.tolgee.constants.Message.IMPORT_LANGUAGE_NOT_FROM_PROJECT) } diff --git a/backend/app/src/main/resources/application-cockroach.yaml b/backend/app/src/main/resources/application-cockroach.yaml new file mode 100644 index 0000000000..84f1e6ba8f --- /dev/null +++ b/backend/app/src/main/resources/application-cockroach.yaml @@ -0,0 +1,9 @@ +spring: + datasource: + url: jdbc:postgresql://cockroachdb:26257/postgres + username: root +tolgee: + database: + type: COCKROACH + postgres-autostart: + enabled: false diff --git a/backend/app/src/main/resources/application-devcockroach.yaml b/backend/app/src/main/resources/application-devcockroach.yaml new file mode 100644 index 0000000000..0ca0de1d63 --- /dev/null +++ b/backend/app/src/main/resources/application-devcockroach.yaml @@ -0,0 +1,3 @@ +spring: + datasource: + url: jdbc:postgresql://localhost:26257/postgres diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/TagsControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/TagsControllerTest.kt index a4eb4b26cc..fbea144c46 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/TagsControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/TagsControllerTest.kt @@ -48,8 +48,8 @@ class TagsControllerTest : ProjectAuthControllerTest("/v2/projects/") { performProjectAuthGet("tags?page=5").andAssertThatJson { node("_embedded.tags") { isArray.hasSize(20) - node("[0].name").isEqualTo("tag 14 12") - node("[19].name").isEqualTo("tag 15 10") + node("[0].name").isEqualTo("tag 05 19") + node("[19].name").isEqualTo("tag 06 18") } } } @@ -61,7 +61,7 @@ class TagsControllerTest : ProjectAuthControllerTest("/v2/projects/") { node("_embedded.tags") { isArray.hasSize(20) node("[0].id").isValidId - node("[0].name").isEqualTo("tag 11 3") + node("[0].name").isEqualTo("tag 02 19") } } } diff --git a/backend/app/src/test/resources/application-cockroach.yaml b/backend/app/src/test/resources/application-cockroach.yaml new file mode 100644 index 0000000000..cd41838663 --- /dev/null +++ b/backend/app/src/test/resources/application-cockroach.yaml @@ -0,0 +1,9 @@ +spring: + datasource: + url: jdbc:postgresql://localhost:26257/postgres + username: root +tolgee: + database: + type: COCKROACH + postgres-autostart: + enabled: false diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityInterceptor.kt b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityInterceptor.kt index 5cdf98d831..cc412b6813 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityInterceptor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityInterceptor.kt @@ -26,6 +26,7 @@ class ActivityInterceptor : EmptyInterceptor() { propertyNames: Array?, types: Array? ): Boolean { + preCommitEventsPublisher.onPersist(entity) interceptedEventsManager.onFieldModificationsActivity( entity, state, null, propertyNames, RevisionType.ADD ) @@ -39,6 +40,7 @@ class ActivityInterceptor : EmptyInterceptor() { propertyNames: Array?, types: Array? ) { + preCommitEventsPublisher.onDelete(entity) interceptedEventsManager.onFieldModificationsActivity( entity, null, state, propertyNames, RevisionType.DEL ) @@ -52,6 +54,7 @@ class ActivityInterceptor : EmptyInterceptor() { propertyNames: Array?, types: Array? ): Boolean { + preCommitEventsPublisher.onUpdate(entity) interceptedEventsManager.onFieldModificationsActivity( entity, currentState, @@ -76,4 +79,7 @@ class ActivityInterceptor : EmptyInterceptor() { val interceptedEventsManager: InterceptedEventsManager get() = applicationContext.getBean(InterceptedEventsManager::class.java) + + val preCommitEventsPublisher: PreCommitEventPublisher + get() = applicationContext.getBean(PreCommitEventPublisher::class.java) } diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/PreCommitEventPublisher.kt b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/PreCommitEventPublisher.kt new file mode 100644 index 0000000000..5d36d5e5d5 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/PreCommitEventPublisher.kt @@ -0,0 +1,23 @@ +package io.tolgee.activity.iterceptor + +import io.tolgee.events.OnEntityPreDelete +import io.tolgee.events.OnEntityPrePersist +import io.tolgee.events.OnEntityPreUpdate +import org.springframework.context.ApplicationContext +import org.springframework.stereotype.Component + +@Component +class PreCommitEventPublisher(private val applicationContext: ApplicationContext) { + + fun onPersist(entity: Any?) { + applicationContext.publishEvent(OnEntityPrePersist(this, entity)) + } + + fun onUpdate(entity: Any?) { + applicationContext.publishEvent(OnEntityPreUpdate(this, entity)) + } + + fun onDelete(entity: Any?) { + applicationContext.publishEvent(OnEntityPreDelete(this, entity)) + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/LiquibaseConfiguration.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/LiquibaseConfiguration.kt index 2bf3bcdf82..6835e58e4f 100644 --- a/backend/data/src/main/kotlin/io/tolgee/configuration/LiquibaseConfiguration.kt +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/LiquibaseConfiguration.kt @@ -1,5 +1,7 @@ package io.tolgee.configuration +import io.tolgee.configuration.tolgee.DatabaseProperties +import io.tolgee.configuration.tolgee.TolgeeProperties import liquibase.integration.spring.SpringLiquibase import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -10,13 +12,15 @@ import javax.sql.DataSource class LiquibaseConfiguration { @Bean @Primary - fun liquibase(dataSource: DataSource): SpringLiquibase { + fun liquibase(dataSource: DataSource, properties: TolgeeProperties): SpringLiquibase { val liquibase = SpringLiquibase() liquibase.dataSource = dataSource liquibase.changeLog = "classpath:db/changelog/schema.xml" liquibase.defaultSchema = "public" - + if (properties.database.type == DatabaseProperties.DatabaseType.COCKROACH) { + liquibase.setChangeLogParameters(mapOf("isCockroach" to "true")) + } return liquibase } } diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/DatabaseProperties.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/DatabaseProperties.kt new file mode 100644 index 0000000000..87347e1ce7 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/DatabaseProperties.kt @@ -0,0 +1,12 @@ +package io.tolgee.configuration.tolgee + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(prefix = "tolgee.database") +class DatabaseProperties { + var type: DatabaseType = DatabaseType.POSTGRES + + enum class DatabaseType { + COCKROACH, POSTGRES + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/TolgeeProperties.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/TolgeeProperties.kt index 8828f21240..2491ce9ae3 100644 --- a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/TolgeeProperties.kt +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/TolgeeProperties.kt @@ -31,5 +31,6 @@ open class TolgeeProperties( var postgresAutostart: PostgresAutostartProperties = PostgresAutostartProperties(), var sendInBlueProperties: SendInBlueProperties = SendInBlueProperties(), open var import: ImportProperties = ImportProperties(), - var rateLimitProperties: RateLimitProperties = RateLimitProperties() + var rateLimitProperties: RateLimitProperties = RateLimitProperties(), + var database: DatabaseProperties = DatabaseProperties() ) diff --git a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/TagsTestData.kt b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/TagsTestData.kt index 28b6d83e3f..6258141509 100644 --- a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/TagsTestData.kt +++ b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/TagsTestData.kt @@ -55,16 +55,18 @@ class TagsTestData : BaseTestData("tagsTestUser", "tagsTestProject") { } } (1..20).forEach { keyNum -> + val keyNumString = keyNum.toString().padStart(2, '0') addKey { - name = "test key $keyNum" + name = "test key $keyNumString" }.build { addMeta { self { (1..20).forEach { tagNum -> + val tagNumString = tagNum.toString().padStart(2, '0') tags.add( Tag().apply { project = projectBuilder.self - name = "tag $keyNum $tagNum" + name = "tag $keyNumString $tagNumString" } ) } diff --git a/backend/data/src/main/kotlin/io/tolgee/events/OnEntityPreDelete.kt b/backend/data/src/main/kotlin/io/tolgee/events/OnEntityPreDelete.kt new file mode 100644 index 0000000000..3a6186476e --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/events/OnEntityPreDelete.kt @@ -0,0 +1,9 @@ +package io.tolgee.events + +import io.tolgee.activity.iterceptor.PreCommitEventPublisher +import org.springframework.context.ApplicationEvent + +class OnEntityPreDelete( + val source: PreCommitEventPublisher, + val entity: Any? +) : ApplicationEvent(source) diff --git a/backend/data/src/main/kotlin/io/tolgee/events/OnEntityPrePersist.kt b/backend/data/src/main/kotlin/io/tolgee/events/OnEntityPrePersist.kt new file mode 100644 index 0000000000..bece6b98a1 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/events/OnEntityPrePersist.kt @@ -0,0 +1,9 @@ +package io.tolgee.events + +import io.tolgee.activity.iterceptor.PreCommitEventPublisher +import org.springframework.context.ApplicationEvent + +class OnEntityPrePersist( + val source: PreCommitEventPublisher, + val entity: Any? +) : ApplicationEvent(source) diff --git a/backend/data/src/main/kotlin/io/tolgee/events/OnEntityPreUpdate.kt b/backend/data/src/main/kotlin/io/tolgee/events/OnEntityPreUpdate.kt new file mode 100644 index 0000000000..3632df6bf7 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/events/OnEntityPreUpdate.kt @@ -0,0 +1,9 @@ +package io.tolgee.events + +import io.tolgee.activity.iterceptor.PreCommitEventPublisher +import org.springframework.context.ApplicationEvent + +class OnEntityPreUpdate( + val source: PreCommitEventPublisher, + val entity: Any? +) : ApplicationEvent(source) diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/activity/ActivityRevisionRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/activity/ActivityRevisionRepository.kt index 4669f24598..c45349b145 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/activity/ActivityRevisionRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/activity/ActivityRevisionRepository.kt @@ -61,7 +61,7 @@ interface ActivityRevisionRepository : JpaRepository { @Query( """ - select count(ar.id) as count, function('to_char', ar.timestamp, 'yyyy-MM-dd') as date + select count(ar.id) as count, cast(cast(ar.timestamp as date) as text) as date from ActivityRevision ar where ar.projectId = :projectId group by date diff --git a/backend/data/src/main/kotlin/io/tolgee/service/LanguageService.kt b/backend/data/src/main/kotlin/io/tolgee/service/LanguageService.kt index cabe60ca0b..ecdaf174d2 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/LanguageService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/LanguageService.kt @@ -16,6 +16,7 @@ import org.springframework.data.domain.Page import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort +import org.springframework.retry.annotation.Retryable import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import java.util.* @@ -78,6 +79,8 @@ class LanguageService( return find(id) ?: throw NotFoundException(Message.LANGUAGE_NOT_FOUND) } + @Transactional + @Retryable() fun find(id: Long): Language? { return languageRepository.findById(id).orElse(null) } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/OrganizationStatsService.kt b/backend/data/src/main/kotlin/io/tolgee/service/OrganizationStatsService.kt index fba08a8857..47456dcc7d 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/OrganizationStatsService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/OrganizationStatsService.kt @@ -1,12 +1,15 @@ package io.tolgee.service +import org.hibernate.FlushMode +import org.hibernate.Session import org.springframework.stereotype.Service import java.math.BigDecimal import javax.persistence.EntityManager +import javax.persistence.FlushModeType @Service class OrganizationStatsService( - private val entityManager: EntityManager + private val entityManager: EntityManager, ) { fun getProjectLanguageCount(projectId: Long): Long { return entityManager @@ -33,9 +36,13 @@ class OrganizationStatsService( } fun getCurrentTranslationCount(organizationId: Long): Long { - val result = entityManager.createNativeQuery( - """ - select + val session = entityManager.unwrap(Session::class.java) + return try { + session.hibernateFlushMode = FlushMode.MANUAL + session.flushMode = FlushModeType.COMMIT + val query = session.createNativeQuery( + """ + select (select sum(keyCount * languageCount) as translationCount from (select p.id as projectId, count(l.id) as languageCount from project as p @@ -47,8 +54,13 @@ class OrganizationStatsService( join key as k on k.project_id = p.id where p.organization_owner_id = :organizationId group by p.id) as keyCounts on keyCounts.projectId = languageCounts.projectId) - """.trimIndent() - ).setParameter("organizationId", organizationId).singleResult as BigDecimal? ?: 0 - return result.toLong() + """.trimIndent() + ).setParameter("organizationId", organizationId) + val result = query.singleResult as BigDecimal? ?: 0 + result.toLong() + } finally { + session.hibernateFlushMode = FlushMode.AUTO + session.flushMode = FlushModeType.AUTO + } } } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/ScreenshotService.kt b/backend/data/src/main/kotlin/io/tolgee/service/ScreenshotService.kt index c362074e59..c11ff65f61 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/ScreenshotService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/ScreenshotService.kt @@ -15,8 +15,11 @@ import io.tolgee.repository.ScreenshotRepository import io.tolgee.security.AuthenticationFacade import io.tolgee.service.ImageUploadService.Companion.UPLOADED_IMAGES_STORAGE_FOLDER_NAME import io.tolgee.util.ImageConverter +import io.tolgee.util.executeInNewTransaction import org.springframework.core.io.InputStreamSource +import org.springframework.retry.annotation.Retryable import org.springframework.stereotype.Service +import org.springframework.transaction.PlatformTransactionManager import org.springframework.transaction.annotation.Transactional @Service @@ -25,13 +28,18 @@ class ScreenshotService( private val fileStorage: FileStorage, private val tolgeeProperties: TolgeeProperties, private val imageUploadService: ImageUploadService, - private val authenticationFacade: AuthenticationFacade + private val authenticationFacade: AuthenticationFacade, + private val transactionManager: PlatformTransactionManager ) { companion object { const val SCREENSHOTS_STORAGE_FOLDER_NAME = "screenshots" } - @Transactional + /** + * CockroachDB has issues with uploading multiple screenshots in the same time, so we + * need to make it retryable and executed in new transaction + */ + @Retryable fun store(screenshotImage: InputStreamSource, key: Key): Screenshot { if (getScreenshotsCountForKey(key) >= tolgeeProperties.maxScreenshotsPerKey) { throw BadRequestException( @@ -42,7 +50,19 @@ class ScreenshotService( val converter = ImageConverter(screenshotImage.inputStream) val image = converter.getImage() val thumbnail = converter.getThumbNail() - return storeProcessed(image.toByteArray(), thumbnail.toByteArray(), key) + var screenshotEntity: Screenshot? = null + try { + return executeInNewTransaction(transactionManager) { + screenshotEntity = storeProcessed(image.toByteArray(), thumbnail.toByteArray(), key) + screenshotEntity!! + } + } catch (e: Exception) { + screenshotEntity?.id?.let { + fileStorage.deleteFile(screenshotEntity!!.getThumbnailPath()) + fileStorage.deleteFile(screenshotEntity!!.getFilePath()) + } + throw e + } } fun storeProcessed(image: ByteArray, thumbnail: ByteArray, key: Key): Screenshot { diff --git a/backend/data/src/main/kotlin/io/tolgee/service/project/ProjectService.kt b/backend/data/src/main/kotlin/io/tolgee/service/project/ProjectService.kt index 62ec8779ae..69fb6a1885 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/project/ProjectService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/project/ProjectService.kt @@ -311,6 +311,7 @@ class ProjectService constructor( projectRepository.saveAll(projects) @CacheEvict(cacheNames = [Caches.PROJECTS], key = "#result.id") + @Transactional fun save(project: Project): Project { val isCreating = project.id == 0L projectRepository.save(project) diff --git a/backend/data/src/main/kotlin/io/tolgee/service/query_builders/TranslationsViewBuilder.kt b/backend/data/src/main/kotlin/io/tolgee/service/query_builders/TranslationsViewBuilder.kt index 5ce8b7c292..1b6bd36e6e 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/query_builders/TranslationsViewBuilder.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/query_builders/TranslationsViewBuilder.kt @@ -1,5 +1,7 @@ package io.tolgee.service.query_builders +import io.tolgee.configuration.tolgee.DatabaseProperties +import io.tolgee.configuration.tolgee.TolgeeProperties import io.tolgee.dtos.request.translation.TranslationFilterByState import io.tolgee.dtos.request.translation.TranslationFilters import io.tolgee.dtos.response.CursorValue @@ -338,9 +340,13 @@ class TranslationsViewBuilder( ): Page { val em = applicationContext.getBean(EntityManager::class.java) val tagService = applicationContext.getBean(TagService::class.java) + val properties = applicationContext.getBean(TolgeeProperties::class.java) + val isCockroachDb = properties.database.type == DatabaseProperties.DatabaseType.COCKROACH - // otherwise it takes forever for postgres to plan the execution - em.createNativeQuery("SET join_collapse_limit TO 1").executeUpdate() + if (!isCockroachDb) { + // otherwise it takes forever for postgres to plan the execution + em.createNativeQuery("SET join_collapse_limit TO 1").executeUpdate() + } var translationsViewBuilder = TranslationsViewBuilder( cb = em.criteriaBuilder, @@ -366,8 +372,10 @@ class TranslationsViewBuilder( } val views = query.resultList.map { KeyWithTranslationsView.of(it, languages.toList()) } - // reset the value - em.createNativeQuery("SET join_collapse_limit TO DEFAULT").executeUpdate() + if (!isCockroachDb) { + // reset the value + em.createNativeQuery("SET join_collapse_limit TO DEFAULT").executeUpdate() + } val keyIds = views.map { it.keyId } tagService.getTagsForKeyIds(keyIds).let { tagMap -> diff --git a/backend/data/src/main/kotlin/io/tolgee/util/withTimeoutAndRetry.kt b/backend/data/src/main/kotlin/io/tolgee/util/withTimeoutAndRetry.kt new file mode 100644 index 0000000000..6219636842 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/util/withTimeoutAndRetry.kt @@ -0,0 +1,28 @@ +package io.tolgee.util + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.async +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout + +fun withTimeoutAndRetry(timeoutInMs: Long = 4000, fn: () -> T): T { + return runBlocking(Dispatchers.IO) { + val retries = 3 + var exp: TimeoutCancellationException? = null + (0..retries).forEach { + val task = async { fn() } + try { + return@runBlocking withTimeout(timeoutInMs) { + task.await() + } + } catch (e: TimeoutCancellationException) { + exp = e + } + } + exp?.let { + throw it + } + throw IllegalStateException("No exception caught!") + } +} diff --git a/backend/data/src/main/resources/db/changelog/schema.xml b/backend/data/src/main/resources/db/changelog/schema.xml index 1914365bf3..336ce696fb 100644 --- a/backend/data/src/main/resources/db/changelog/schema.xml +++ b/backend/data/src/main/resources/db/changelog/schema.xml @@ -82,6 +82,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -444,6 +482,14 @@ + + + + + + select setval('hibernate_sequence', 1000000000) + + @@ -1088,6 +1134,11 @@ + + + + + @@ -1098,6 +1149,20 @@ + + + + + + + + + + + + + + @@ -1357,6 +1422,11 @@ + + + + + @@ -1399,6 +1469,11 @@ constraintName="organization_third_party_billing_id_unique" tableName="organization"/> + + + + + + + + + + diff --git a/backend/testing/src/main/kotlin/io/tolgee/CleanDbTestListener.kt b/backend/testing/src/main/kotlin/io/tolgee/CleanDbTestListener.kt index d25bdc0ad6..eb6e4e5213 100644 --- a/backend/testing/src/main/kotlin/io/tolgee/CleanDbTestListener.kt +++ b/backend/testing/src/main/kotlin/io/tolgee/CleanDbTestListener.kt @@ -5,6 +5,7 @@ import org.slf4j.LoggerFactory import org.springframework.context.ApplicationContext import org.springframework.test.context.TestContext import org.springframework.test.context.TestExecutionListener +import java.sql.Connection import java.sql.ResultSet import javax.sql.DataSource import kotlin.system.measureTimeMillis @@ -47,7 +48,7 @@ class CleanDbTestListener : TestExecutionListener { val ds: DataSource = appContext.getBean(DataSource::class.java) ds.connection.use { conn -> val stmt = conn.createStatement() - val databaseName: Any = "postgres" + val databaseName: Any = getDatabaseName(conn) val ignoredTablesString = ignoredTables.joinToString(", ") { "'$it'" } val rs: ResultSet = stmt.executeQuery( String.format( @@ -71,6 +72,15 @@ class CleanDbTestListener : TestExecutionListener { } } + private fun getDatabaseName(conn: Connection): String { + val rs = conn.getMetaData().catalogs + val data = mutableListOf() + while (rs.next()) { + data.add(rs.getString(1)) + } + return data.single() + } + @Throws(Exception::class) override fun afterTestMethod(testContext: TestContext) { } diff --git a/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractTransactionalTest.kt b/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractTransactionalTest.kt index 75cd912aa6..829223fee5 100644 --- a/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractTransactionalTest.kt +++ b/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractTransactionalTest.kt @@ -2,7 +2,6 @@ package io.tolgee.testing import io.tolgee.CleanDbTestListener import org.springframework.beans.factory.annotation.Autowired -import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.TestExecutionListeners import org.springframework.test.context.support.DependencyInjectionTestExecutionListener import org.springframework.test.context.transaction.TestTransaction @@ -16,7 +15,6 @@ import javax.persistence.EntityManager CleanDbTestListener::class ] ) -@ActiveProfiles(profiles = ["local"]) abstract class AbstractTransactionalTest { @Autowired protected lateinit var entityManager: EntityManager diff --git a/build.gradle b/build.gradle index a27640244c..a6c776cc54 100644 --- a/build.gradle +++ b/build.gradle @@ -147,4 +147,14 @@ task ktlintFormat(type: JavaExec, group: "formatting") { subprojects { task allDeps(type: DependencyReportTask) {} + + task startCockroachDb(type: Exec){ + onlyIf { System.getenv("SPRING_PROFILES_ACTIVE") == "cockroach" } + commandLine "bash", "-c", "docker run --rm -d --name=roach -p 26257:26257 -p 8089:8080 cockroachdb/cockroach:v22.2.1 start-single-node --insecure --store=type=mem,size=0.25" + } + + task stopCockroachDb(type: Exec){ + onlyIf { System.getenv("SPRING_PROFILES_ACTIVE") == "cockroach" } + commandLine "bash", "-c", "docker stop roach" + } } diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index 702d316db8..8e7466cd03 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -7,7 +7,7 @@ services: - 8201:8201 - 8091:8091 environment: - - spring.profiles.active=docker,e2e + - spring.profiles.active=docker,e2e,${SPRING_PROFILES_ACTIVE:-default} - tolgee.smtp.host=fakesmtp - tolgee.smtp.port=1025 - tolgee.frontend-url=http://localhost:8201 @@ -18,5 +18,15 @@ services: ports: - "21025:1025" - "21080:1080" + cockroachdb: + container_name: tolgee_cockroach_e2e + image: cockroachdb/cockroach:v22.2.0 + ports: + - 26257:26257 + - 8088:8080 + command: + - start-single-node + - --insecure + - --store=type=mem,size=0.25 volumes: e2e-db-data: diff --git a/gradle/e2e.gradle b/gradle/e2e.gradle index fd40094d9f..20bfc8f7a4 100644 --- a/gradle/e2e.gradle +++ b/gradle/e2e.gradle @@ -6,6 +6,7 @@ ext { E2E_DIR = "${project.projectDir}/e2e" BILLING_E2E_DIR = "${project.projectDir}/../billing/e2e" WEBAPP_DIR = "${project.projectDir}/webapp" + IS_COCKROACH = System.getenv("SPRING_PROFILES_ACTIVE")?.contains("cockroach") } @@ -80,7 +81,7 @@ task runE2e(type: Exec, group: "e2e") { finalizedBy "saveServerLogs", "stopDockerE2e", "cleanupDockerE2e" } -task saveServerLogs(type: Exec, group: "e2E"){ +task saveServerLogs(type: Exec, group: "e2E") { commandLine "bash", "-c", "docker-compose logs > server.log" workingDir E2E_DIR } @@ -122,13 +123,21 @@ task runWebAppNpmStartE2eDev(type: Exec, group: "e2e") { task runDockerE2e(type: Exec, group: "e2e") { dependsOn "tagDockerLocal" - commandLine "docker-compose", "up", "-d" + if (IS_COCKROACH) { + commandLine "docker-compose", "up", "-d", "fakesmtp", "cockroachdb", "app" + } else { + commandLine "docker-compose", "up", "-d", "fakesmtp", "app" + } workingDir E2E_DIR finalizedBy "waitForRunningContainer" } task runDockerE2eDev(type: Exec, group: "e2e") { - commandLine "docker-compose", "up", "-d", "fakesmtp" + if (IS_COCKROACH) { + commandLine "docker-compose", "up", "-d", "fakesmtp", "cockroachdb" + } else { + commandLine "docker-compose", "up", "-d", "fakesmtp" + } workingDir E2E_DIR }