Skip to content

Commit 1aefc36

Browse files
naviktthomasnaviktthomas
authored
Merge pull request #131 from navikt/dev/arbeidssoeker-synk-jobb
Dev/arbeidssoeker synk jobb
2 parents b31f411 + f586d45 commit 1aefc36

File tree

18 files changed

+374
-0
lines changed

18 files changed

+374
-0
lines changed

‎.github/workflows/arbeidssoekere-synk-jobb.yaml

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
name: Arbeidssøkere Synk Jobb
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
- dev/*
8+
paths:
9+
- 'jobs/arbeidssoekere-synk-jobb/**'
10+
- 'lib/**'
11+
- 'domain/**'
12+
- 'arbeidssoekere-synk-jobb.yaml'
13+
- 'gradle/**'
14+
- 'settings.gradle.kts'
15+
- 'gradle.properties'
16+
- 'gradlew'
17+
- 'gradlew.bat'
18+
19+
env:
20+
MODULE: arbeidssoekere-synk-jobb
21+
IMAGE: europe-north1-docker.pkg.dev/${{ vars.NAIS_MANAGEMENT_PROJECT_ID }}/paw/paw-arbeidssoekere-synk-jobb
22+
jobs:
23+
build:
24+
name: Build
25+
runs-on: ubuntu-latest
26+
timeout-minutes: 10
27+
permissions:
28+
contents: read
29+
id-token: write
30+
packages: write
31+
outputs:
32+
image: ${{ steps.docker-build-push.outputs.image }}
33+
steps:
34+
- name: Checkout
35+
uses: actions/checkout@v4
36+
- name: Setup Java
37+
uses: actions/setup-java@v4
38+
with:
39+
java-version: 21
40+
distribution: temurin
41+
cache: gradle
42+
- name: Set version
43+
run: echo "VERSION=$(date +'%y.%m.%d').${{ github.run_number }}-${{ github.run_attempt }}" >> $GITHUB_ENV
44+
- name: Login GAR
45+
uses: nais/login@v0
46+
with:
47+
project_id: ${{ vars.NAIS_MANAGEMENT_PROJECT_ID }}
48+
identity_provider: ${{ secrets.NAIS_WORKLOAD_IDENTITY_PROVIDER }}
49+
team: paw
50+
- name: Build and push image with Gradle
51+
id: docker-build-push
52+
working-directory: ./
53+
env:
54+
ORG_GRADLE_PROJECT_githubPassword: ${{ secrets.GITHUB_TOKEN }}
55+
run: |
56+
echo "image=${{ env.IMAGE }}:${{ env.VERSION }}" >> $GITHUB_OUTPUT
57+
echo -Pversion=${{ env.VERSION }} -Pimage=${{ env.IMAGE }} :jobs:${{ env.MODULE }}:build :jobs:${{ env.MODULE }}:jib
58+
./gradlew -Pversion=${{ env.VERSION }} -Pimage=${{ env.IMAGE }} :jobs:${{ env.MODULE }}:build :jobs:${{ env.MODULE }}:jib
59+
echo "DIGEST=$(cat ./jobs/${{ env.MODULE }}/build/jib-image.digest)" >> $GITHUB_ENV
60+
- name: Attest and sign image
61+
uses: nais/[email protected]
62+
with:
63+
image_ref: ${{ env.IMAGE }}@${{ env.DIGEST }}
64+
65+
deploy-dev:
66+
if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/dev')
67+
name: Deploy to dev-gcp
68+
needs:
69+
- build
70+
permissions:
71+
contents: read
72+
id-token: write
73+
runs-on: ubuntu-latest
74+
steps:
75+
- name: Checkout
76+
uses: actions/checkout@v4
77+
- name: Deploy to GCP
78+
uses: nais/deploy/actions/deploy@v2
79+
env:
80+
CLUSTER: dev-gcp
81+
RESOURCE: ./jobs/${{ env.MODULE }}/nais/nais-dev.yaml
82+
VAR: image=${{ needs.build.outputs.image }}

‎gradle/libs.versions.toml

+1
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ avro-core = { group = "org.apache.avro", name = "avro", version.ref = "orgApache
7676
avro-kafkaSerializer = { group = "io.confluent", name = "kafka-avro-serializer", version.ref = "ioConfluentKafkaVersion" }
7777
avro-kafkaStreamsSerde = { group = "io.confluent", name = "kafka-streams-avro-serde", version.ref = "ioConfluentKafkaVersion" }
7878
jackson-datatypeJsr310 = { group = "com.fasterxml.jackson.datatype", name = "jackson-datatype-jsr310", version.ref = "comFasterxmlJacksonVersion" }
79+
jackson-dataformat-csv = { group = "com.fasterxml.jackson.dataformat", name = "jackson-dataformat-csv", version.ref = "comFasterxmlJacksonVersion" }
7980
jackson-kotlin = { group = "com.fasterxml.jackson.module", name = "jackson-module-kotlin", version.ref = "comFasterxmlJacksonVersion" }
8081
jackson-core = { group = "com.fasterxml.jackson.core", name = "jackson-core", version.ref = "comFasterxmlJacksonVersion" }
8182
graphql-client = { group = "com.expediagroup", name = "graphql-kotlin-client", version.ref = "graphqlClientVersion" }

‎jobs/arbeidssoekere-synk-jobb/README.md

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# paw-arbeidssoeker-synk-jobb
2+
3+
Jobb for å synke arbeidssøkere i arbeidssøkerregisteret.
4+
5+
## CSV Fil
6+
Jobben forventer å finne en CSV-fil under stien som er definert av `mountPath` variabelen i `job_config.toml`
7+
konfig-filen.
8+
9+
> [!WARNING]
10+
> Siden CSV-filen inneholder persondata så skal filen opprettes som en secret i Kubernetes
11+
12+
### Opprette CSV-fil som secret i Kubernetes
13+
Informasjon om secrets fra [NAIS docs](https://docs.nais.io/services/secrets/).
14+
15+
```shell
16+
kubectl create secret generic paw-arbeidssoekere-csv --from-file=v1.csv=/tmp/paw-arbeidssoekere-csv/v1.csv
17+
```

‎jobs/arbeidssoekere-synk-jobb/build.gradle.kts

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
plugins {
2+
kotlin("jvm")
3+
id("jib-distroless")
4+
application
5+
}
6+
7+
val jvmMajorVersion: String by project
8+
9+
dependencies {
10+
// Project
11+
implementation(project(":lib:hoplite-config"))
12+
implementation(project(":lib:logging"))
13+
14+
// Logging
15+
implementation(libs.nav.common.log)
16+
17+
// Jackson
18+
implementation(libs.jackson.kotlin)
19+
implementation(libs.jackson.dataformat.csv)
20+
21+
// Test
22+
testImplementation(libs.bundles.testLibsWithUnitTesting)
23+
}
24+
25+
java {
26+
toolchain {
27+
languageVersion.set(JavaLanguageVersion.of(jvmMajorVersion))
28+
}
29+
}
30+
31+
application {
32+
mainClass.set("no.nav.paw.arbeidssoeker.synk.JobKt")
33+
}
34+
35+
tasks.withType<Test>().configureEach {
36+
useJUnitPlatform()
37+
}
38+
39+
tasks.withType(Jar::class) {
40+
manifest {
41+
attributes["Implementation-Version"] = project.version
42+
attributes["Main-Class"] = application.mainClass.get()
43+
attributes["Implementation-Title"] = rootProject.name
44+
}
45+
}

‎jobs/arbeidssoekere-synk-jobb/nais/nais-dev.yaml

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
apiVersion: nais.io/v1
2+
kind: Naisjob
3+
metadata:
4+
name: paw-arbeidssoekere-synk-jobb
5+
namespace: paw
6+
labels:
7+
team: paw
8+
spec:
9+
image: {{ image }}
10+
resources:
11+
requests:
12+
cpu: 200m
13+
memory: 256Mi
14+
secureLogs:
15+
enabled: true
16+
filesFrom:
17+
- secret: paw-arbeidssoekere-csv
18+
mountPath: /var/run/secrets/paw-arbeidssoekere-csv

‎jobs/arbeidssoekere-synk-jobb/src/main/kotlin/no/nav/paw/arbeidssoeker/synk/Job.kt

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package no.nav.paw.arbeidssoeker.synk
2+
3+
import no.nav.paw.arbeidssoeker.synk.config.JOB_CONFIG
4+
import no.nav.paw.arbeidssoeker.synk.config.JobConfig
5+
import no.nav.paw.arbeidssoeker.synk.service.SyncService
6+
import no.nav.paw.config.env.appNameOrDefaultForLocal
7+
import no.nav.paw.config.hoplite.loadNaisOrLocalConfiguration
8+
import no.nav.paw.logging.logger.buildApplicationLogger
9+
10+
fun main() {
11+
val logger = buildApplicationLogger
12+
val jobConfig = loadNaisOrLocalConfiguration<JobConfig>(JOB_CONFIG)
13+
val syncService = SyncService(jobConfig)
14+
15+
val name = jobConfig.runtimeEnvironment.appNameOrDefaultForLocal(default = "local-job")
16+
17+
try {
18+
logger.info("Starter $name")
19+
syncService.syncArbeidssoekere()
20+
} catch (throwable: Throwable) {
21+
logger.error("Kjøring feilet", throwable)
22+
} finally {
23+
logger.info("Avslutter $name")
24+
}
25+
}

‎jobs/arbeidssoekere-synk-jobb/src/main/kotlin/no/nav/paw/arbeidssoeker/synk/config/JobConfig.kt

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package no.nav.paw.arbeidssoeker.synk.config
2+
3+
import no.nav.paw.config.env.RuntimeEnvironment
4+
import no.nav.paw.config.env.currentRuntimeEnvironment
5+
6+
const val JOB_CONFIG = "job_config.toml"
7+
8+
data class JobConfig(
9+
val mountPath: String,
10+
val runtimeEnvironment: RuntimeEnvironment = currentRuntimeEnvironment
11+
)

‎jobs/arbeidssoekere-synk-jobb/src/main/kotlin/no/nav/paw/arbeidssoeker/synk/model/Arbeidssoeker.kt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
package no.nav.paw.arbeidssoeker.synk.model
2+
3+
data class Arbeidssoeker(val identitetsnummer: String)

‎jobs/arbeidssoekere-synk-jobb/src/main/kotlin/no/nav/paw/arbeidssoeker/synk/service/SyncService.kt

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
package no.nav.paw.arbeidssoeker.synk.service
2+
3+
import no.nav.paw.arbeidssoeker.synk.config.JobConfig
4+
import no.nav.paw.arbeidssoeker.synk.utils.ArbeidssoekerCsvReader
5+
import no.nav.paw.logging.logger.buildApplicationLogger
6+
import no.nav.paw.logging.logger.buildNamedLogger
7+
import java.io.File
8+
import java.nio.file.Files
9+
import java.time.Duration
10+
import java.time.Instant
11+
import kotlin.io.path.Path
12+
13+
class SyncService(
14+
private val jobConfig: JobConfig
15+
) {
16+
private val logger = buildApplicationLogger
17+
private val secureLogger = buildNamedLogger("secure")
18+
19+
fun syncArbeidssoekere() {
20+
var count = 0
21+
val timestamp = Instant.now()
22+
val file = getFile()
23+
logger.info("Leser CSV-fil fra {}", file.absolutePath)
24+
val values = ArbeidssoekerCsvReader.readValues(file)
25+
logger.info("Starter prosessering av CSV-data")
26+
while (values.hasNextValue()) {
27+
count++
28+
if (count % 1000 == 0) {
29+
logger.info("Prosessert {} linjer CSV-data på {} ms", count, timestamp.millisSince())
30+
}
31+
val value = values.nextValue()
32+
secureLogger.info("Prosesserer arbeidssøker {}", value.identitetsnummer)
33+
}
34+
logger.info("Fullførte prosessering av {} linjer CSV-data på {} ms", count, timestamp.millisSince())
35+
}
36+
37+
private fun getFile(): File {
38+
with(jobConfig) {
39+
val path = Path(mountPath)
40+
if (!Files.exists(path)) {
41+
throw IllegalStateException("$mountPath ikke funnet")
42+
}
43+
if (!Files.isRegularFile(path)) {
44+
throw IllegalStateException("$mountPath er ikke en fil")
45+
}
46+
if (!Files.isReadable(path)) {
47+
throw IllegalStateException("$mountPath kan ikke leses fra")
48+
}
49+
return path.toFile()
50+
}
51+
}
52+
53+
private fun Instant.millisSince(): Long = Duration.between(this, Instant.now()).toMillis()
54+
}

‎jobs/arbeidssoekere-synk-jobb/src/main/kotlin/no/nav/paw/arbeidssoeker/synk/utils/CsvReader.kt

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package no.nav.paw.arbeidssoeker.synk.utils
2+
3+
import com.fasterxml.jackson.databind.MappingIterator
4+
import com.fasterxml.jackson.databind.ObjectMapper
5+
import com.fasterxml.jackson.databind.ObjectReader
6+
import com.fasterxml.jackson.dataformat.csv.CsvMapper
7+
import com.fasterxml.jackson.dataformat.csv.CsvSchema
8+
import com.fasterxml.jackson.module.kotlin.KotlinModule
9+
import no.nav.paw.arbeidssoeker.synk.model.Arbeidssoeker
10+
import java.io.File
11+
import java.net.URL
12+
13+
private val csvMapper: ObjectMapper = CsvMapper()
14+
.registerModule(KotlinModule.Builder().build())
15+
private val csvSchema: CsvSchema = CsvSchema.builder()
16+
.setAllowComments(true)
17+
.setColumnSeparator(',')
18+
.setUseHeader(true)
19+
.build()
20+
21+
sealed class CsvReader<T>(val objectReader: ObjectReader) {
22+
fun readValues(file: File): MappingIterator<T> = objectReader.readValues(file)
23+
fun readValues(url: URL): MappingIterator<T> = objectReader.readValues(url)
24+
}
25+
26+
data object ArbeidssoekerCsvReader : CsvReader<Arbeidssoeker>(
27+
objectReader = csvMapper
28+
.readerFor(Arbeidssoeker::class.java)
29+
.with(csvSchema)
30+
)

‎jobs/arbeidssoekere-synk-jobb/src/main/resources/local/database_config.toml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
host = "localhost"
2+
port = 5432
3+
username = "paw_arbeidssoekere_synk_jobb"
4+
password = "Paw1234"
5+
database = "paw_arbeidssoekere"

‎jobs/arbeidssoekere-synk-jobb/src/main/resources/local/job_config.toml

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
mountPath = "/tmp/paw-arbeidssoekere-csv/v1.csv"

‎jobs/arbeidssoekere-synk-jobb/src/main/resources/logback.xml

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<configuration>
3+
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
4+
<encoder>
5+
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %5level %logger{36} - %msg%n</pattern>
6+
</encoder>
7+
</appender>
8+
<appender name="STDOUT_JSON" class="ch.qos.logback.core.ConsoleAppender">
9+
<encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
10+
</appender>
11+
<appender name="SECURE_LOG_JSON" class="ch.qos.logback.core.rolling.RollingFileAppender">
12+
<file>/secure-logs/secure.log</file>
13+
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
14+
<fileNamePattern>/secure-logs/secure.log.%i</fileNamePattern>
15+
<minIndex>1</minIndex>
16+
<maxIndex>1</maxIndex>
17+
</rollingPolicy>
18+
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
19+
<maxFileSize>50MB</maxFileSize>
20+
</triggeringPolicy>
21+
<encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
22+
</appender>
23+
24+
<if condition='"${NAIS_CLUSTER_NAME}" == "prod-gcp"'>
25+
<then>
26+
<root level="INFO">
27+
<appender-ref ref="STDOUT_JSON"/>
28+
</root>
29+
<logger name="no.nav.paw.logger.secure" level="INFO" additivity="false">
30+
<appender-ref ref="SECURE_LOG_JSON"/>
31+
</logger>
32+
</then>
33+
</if>
34+
<if condition='"${NAIS_CLUSTER_NAME}" == "dev-gcp"'>
35+
<then>
36+
<root level="INFO">
37+
<appender-ref ref="STDOUT_JSON"/>
38+
</root>
39+
<logger name="no.nav" level="DEBUG"/>
40+
<logger name="no.nav.paw.logger.secure" level="INFO" additivity="false">
41+
<appender-ref ref="SECURE_LOG_JSON"/>
42+
</logger>
43+
</then>
44+
</if>
45+
<if condition='"${NAIS_CLUSTER_NAME}" == "NAIS_CLUSTER_NAME_IS_UNDEFINED"'>
46+
<then>
47+
<root level="INFO">
48+
<appender-ref ref="STDOUT"/>
49+
</root>
50+
<logger name="no.nav" level="DEBUG"/>
51+
</then>
52+
</if>
53+
</configuration>

‎jobs/arbeidssoekere-synk-jobb/src/main/resources/nais/database_config.toml

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
jdbcUrl = "${NAIS_DATABASE_PAW_ARBEIDSSOEKERE_SYNK_JOBB_PAW_ARBEIDSSOEKERE_JDBC_URL}"

‎jobs/arbeidssoekere-synk-jobb/src/main/resources/nais/job_config.toml

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
mountPath = "/var/run/secrets/paw-arbeidssoekere-csv/v1.csv"

‎jobs/arbeidssoekere-synk-jobb/src/test/kotlin/no/nav/paw/arbeidssoeker/synk/utils/CsvReaderTest.kt

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package no.nav.paw.arbeidssoeker.synk.utils
2+
3+
import io.kotest.core.spec.style.FreeSpec
4+
5+
class CsvReaderTest : FreeSpec({
6+
"Skal lese CSV-fil" {
7+
val url = javaClass.getResource("/sync.csv")!!
8+
val values = ArbeidssoekerCsvReader.readValues(url)
9+
while (values.hasNextValue()) {
10+
val arbeidssoeker = values.nextValue()
11+
println(arbeidssoeker.identitetsnummer)
12+
}
13+
}
14+
})

‎jobs/arbeidssoekere-synk-jobb/src/test/resources/sync.csv

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# version v1
2+
identitetsnummer
3+
01017012345
4+
02017012345
5+
03017012345
6+
04017012345
7+
05017012345

0 commit comments

Comments
 (0)