Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,19 @@ variables:
SBT_OPTS: "-sbt-launch-dir .sbt/launchers -sbt-dir .sbt -ivy .ivy2 -Dsbt.color=true -Dscala.color=true"
SERVICE: treehub

image: registry.gitlab.com/torizon-platform/ci-container-build:scala-jvm17-0.0.5
image: registry.gitlab.com/toradex/torizon-cloud/ci-container-build/scala:4acf34bc

include:
- project: torizon-platform/ci-container-build
- project: toradex/torizon-cloud/ci-container-build
file: gitlab-ci-templates/gitlab-sbt-docker-publish.yaml
ref: master
- project: 'torizon-platform/deploy-env'
- project: 'toradex/torizon-cloud/deploy-env'
ref: master
file: '/gitlab/deploy-core-service.yml'

services:
- name: docker:dind
- name: mariadb:10.4
- name: docker:28.5.0-dind
- name: mariadb:10.11
alias: db
command:
- --character-set-server=utf8
Expand Down Expand Up @@ -54,6 +54,8 @@ test:
- mysql -v -h db -u root -proot < db_user.sql
script:
- sbt ut:test
tags:
- torizon-cloud-team

build docker image:
stage: build image
Expand All @@ -63,3 +65,5 @@ build docker image:
extends: .sbt-docker-publish
variables:
SERVICE_NAME: treehub
tags:
- torizon-cloud-team
22 changes: 11 additions & 11 deletions build.sbt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name := "treehub"
organization := "io.github.uptane"
scalaVersion := "2.13.11"
scalaVersion := "2.13.16"

scalacOptions := Seq("-unchecked", "-deprecation", "-encoding", "utf8", "-Xasync", "-Xsource:3")

Expand All @@ -21,18 +21,18 @@ lazy val treehub = (project in file("."))
.settings(UnitTest / testOptions := Seq(Tests.Filter(unitFilter)))
.settings(IntegrationTest / testOptions := Seq(Tests.Filter(itFilter)))
.settings(Seq(libraryDependencies ++= {
val akkaV = "2.6.20"
val akkaHttpV = "10.2.10"
val pekkoV = "1.1.5"
val pekkoHttpV = "1.2.0"
val scalaTestV = "3.0.9"
val libatsV = "2.1.2"
val libatsV = "5.0.0"

Seq(
"com.typesafe.akka" %% "akka-actor" % akkaV,
"com.typesafe.akka" %% "akka-stream" % akkaV,
"com.typesafe.akka" %% "akka-stream-testkit" % akkaV % "test",
"com.typesafe.akka" %% "akka-http" % akkaHttpV,
"com.typesafe.akka" %% "akka-http-testkit" % akkaHttpV % "test",
"com.typesafe.akka" %% "akka-slf4j" % akkaV,
"org.apache.pekko" %% "pekko-actor" % pekkoV,
"org.apache.pekko" %% "pekko-stream" % pekkoV,
"org.apache.pekko" %% "pekko-stream-testkit" % pekkoV % "test",
"org.apache.pekko" %% "pekko-http" % pekkoHttpV,
"org.apache.pekko" %% "pekko-http-testkit" % pekkoHttpV % "test",
"org.apache.pekko" %% "pekko-slf4j" % pekkoV,
"org.scalatest" %% "scalatest" % scalaTestV % "test,it",

"ch.qos.logback" % "logback-classic" % "1.5.6",
Expand All @@ -44,7 +44,7 @@ lazy val treehub = (project in file("."))
"io.github.uptane" %% "libats-messaging" % libatsV,
"io.github.uptane" %% "libats-messaging-datatype" % libatsV,
"io.github.uptane" %% "libats-slick" % libatsV,
"io.github.uptane" %% "libats-metrics-akka" % libatsV,
"io.github.uptane" %% "libats-metrics-pekko" % libatsV,
"io.github.uptane" %% "libats-metrics-prometheus" % libatsV,
"io.github.uptane" %% "libats-logging" % libatsV,
"io.github.uptane" %% "libats-logging" % libatsV,
Expand Down
10 changes: 5 additions & 5 deletions deploy/ci-db-setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,16 @@

set -xeuo pipefail

MYSQL_COMMAND=mysql
MYSQL_COMMAND=mariadb
HOST=$1
MYSQL=mysql
MYSQLADMIN=mysqladmin
MYSQL=mariadb
MYSQLADMIN=mariadb-admin

until $MYSQLADMIN ping --silent --protocol=TCP -h $HOST -P 3306 -u root -proot; do
until $MYSQLADMIN ping --skip-ssl --silent --protocol=TCP -h $HOST -P 3306 -u root -proot; do
echo waiting for mysql; sleep 1
done

$MYSQL -v -h $HOST -u root -proot <<EOF
$MYSQL --skip-ssl -v -h $HOST -u root -proot <<EOF
CREATE DATABASE IF NOT EXISTS ota_treehub;

CREATE USER IF NOT EXISTS 'treehub' IDENTIFIED BY 'treehub';
Expand Down
12 changes: 6 additions & 6 deletions src/main/resources/application.conf
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
akka {
loggers = ["akka.event.slf4j.Slf4jLogger"]
pekko {
loggers = ["org.apache.pekko.event.slf4j.Slf4jLogger"]

http.server = {
parsing.max-content-length = 3GB
parsing.max-content-length = ${?AKKA_HTTP_MAX_CONTENT_LENGTH}
parsing.max-content-length = ${?PEKKO_HTTP_MAX_CONTENT_LENGTH}
request-timeout = 5 minutes
request-timeout = ${?AKKA_HTTP_REQUEST_TIMEOUT}
request-timeout = ${?PEKKO_HTTP_REQUEST_TIMEOUT}
idle-timeout = 5 minutes
idle-timeout = ${?AKKA_HTTP_IDLE_TIMEOUT}
idle-timeout = ${?PEKKO_HTTP_IDLE_TIMEOUT}
max-connections = 1024
max-connections = ${?AKKA_HTTP_MAX_CONNECTIONS}
max-connections = ${?PEKKO_HTTP_MAX_CONNECTIONS}
}

actor.deployment {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ package com.advancedtelematic.common

import java.security.MessageDigest

import akka.stream.scaladsl.Sink
import akka.util.ByteString
import org.apache.pekko.stream.scaladsl.Sink
import org.apache.pekko.util.ByteString

import scala.concurrent.{ExecutionContext, Future}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,7 @@ object GVariantEncoder {

buf ++= ByteVector.fill(8 - buf.length % 8)(0)

buf ++= ByteVector.fromValidHex(superblockHash.value) ++ ByteVector.fromByte(0)
++ ByteVector("ay".getBytes()) ++ ByteVector.fromByte(keyEndsAt.toByte)
buf ++= ByteVector.fromValidHex(superblockHash.value) ++ ByteVector.fromByte(0) ++ ByteVector("ay".getBytes()) ++ ByteVector.fromByte(keyEndsAt.toByte)

val offsetBeforePadding = buf.length

Expand Down
14 changes: 7 additions & 7 deletions src/main/scala/com/advancedtelematic/treehub/Boot.scala
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package com.advancedtelematic.treehub

import akka.actor.ActorSystem
import akka.http.scaladsl.Http
import akka.http.scaladsl.Http.ServerBinding
import akka.http.scaladsl.server.{Directives, Route}
import org.apache.pekko.actor.ActorSystem
import org.apache.pekko.http.scaladsl.Http
import org.apache.pekko.http.scaladsl.Http.ServerBinding
import org.apache.pekko.http.scaladsl.server.{Directives, Route}
import com.advancedtelematic.libats.http.LogDirectives.*
import com.advancedtelematic.libats.http.VersionDirectives.*
import com.advancedtelematic.libats.http.tracing.Tracing
Expand All @@ -12,7 +12,7 @@ import com.advancedtelematic.libats.messaging.MessageBus
import com.advancedtelematic.libats.slick.db.{BootMigrations, CheckMigrations, DatabaseSupport}
import com.advancedtelematic.libats.slick.monitoring.DatabaseMetrics
import com.advancedtelematic.metrics.prometheus.PrometheusMetricsSupport
import com.advancedtelematic.metrics.{AkkaHttpConnectionMetrics, AkkaHttpRequestMetrics, MetricsSupport}
import com.advancedtelematic.metrics.{PekkoHttpConnectionMetrics, PekkoHttpRequestMetrics, MetricsSupport}
import com.advancedtelematic.treehub.daemon.{DeletedDeltaCleanupActor, StaleObjectArchiveActor}
import com.advancedtelematic.treehub.delta_store.StaticDeltas
import com.advancedtelematic.treehub.http.TreeHubRoutes
Expand All @@ -35,8 +35,8 @@ class TreehubBoot(override val globalConfig: Config,
with DatabaseSupport
with MetricsSupport
with DatabaseMetrics
with AkkaHttpRequestMetrics
with AkkaHttpConnectionMetrics
with PekkoHttpRequestMetrics
with PekkoHttpConnectionMetrics
with PrometheusMetricsSupport
with CheckMigrations
with VersionInfo
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package com.advancedtelematic.treehub

import java.nio.file.Paths

import akka.http.scaladsl.model.Uri
import org.apache.pekko.http.scaladsl.model.Uri
import com.advancedtelematic.treehub.object_store.S3Credentials
import com.amazonaws.regions.Regions
import com.typesafe.config.ConfigFactory
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package com.advancedtelematic.treehub.daemon

import akka.actor.{Actor, ActorLogging, Props}
import akka.actor.Status.Failure
import akka.pattern.{BackoffOpts, BackoffSupervisor}
import org.apache.pekko.actor.{Actor, ActorLogging, Props}
import org.apache.pekko.actor.Status.Failure
import org.apache.pekko.pattern.{BackoffOpts, BackoffSupervisor}
import com.advancedtelematic.data.DataType.StaticDeltaMeta
import com.advancedtelematic.treehub.daemon.DeletedDeltaCleanupActor.{Done, Tick, defaultTickInterval}
import com.advancedtelematic.treehub.db.StaticDeltaMetaRepositorySupport
Expand All @@ -29,7 +29,7 @@ class DeletedDeltaCleanupActor(storage: BlobStore, tickInterval: FiniteDuration
with StaticDeltaMetaRepositorySupport
with ActorLogging {

import akka.pattern.pipe
import org.apache.pekko.pattern.pipe

import scala.async.Async.*

Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
package com.advancedtelematic.treehub.daemon

import java.time.Instant
import akka.actor.Status.Failure
import akka.actor.{Actor, ActorLogging, Props}
import akka.pattern.{BackoffOpts, BackoffSupervisor}
import akka.stream.Materializer
import akka.stream.scaladsl.{Flow, Sink}
import org.apache.pekko.actor.Status.Failure
import org.apache.pekko.actor.{Actor, ActorLogging, Props}
import org.apache.pekko.pattern.{BackoffOpts, BackoffSupervisor}
import org.apache.pekko.stream.Materializer
import org.apache.pekko.stream.scaladsl.{Flow, Sink}
import com.advancedtelematic.data.DataType.{ObjectId, ObjectStatus, TObject}
import com.advancedtelematic.treehub.daemon.StaleObjectArchiveActor.{Done, Tick}
import com.advancedtelematic.treehub.db.{ArchivedObjectRepositorySupport, ObjectRepositorySupport}
Expand Down Expand Up @@ -35,7 +35,7 @@ class StaleObjectArchiveActor(blobStore: BlobStore, objectsExpireAfter: java.tim
with ArchivedObjectRepositorySupport
with ActorLogging {

import akka.pattern.pipe
import org.apache.pekko.pattern.pipe
import context.dispatcher

import scala.async.Async.*
Expand Down
10 changes: 7 additions & 3 deletions src/main/scala/com/advancedtelematic/treehub/db/DbOps.scala
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
package com.advancedtelematic.treehub.db

import com.advancedtelematic.libats.data.PaginationResult.{Limit, Offset}

object DbOps {
implicit class PaginationResultOps(x: Option[Long]) {
def orDefaultOffset: Long = x.getOrElse(0L)
implicit class PaginationResultOffsetOps(x: Option[Offset]) {
def orDefaultOffset: Offset = x.getOrElse(Offset(0))
}

def orDefaultLimit: Long = x.getOrElse(50L)
implicit class PaginationResultLimitOps(x: Option[Limit]) {
def orDefaultLimit: Limit = x.getOrElse(Limit(50L))
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ package com.advancedtelematic.treehub.db

import java.time.Instant

import akka.NotUsed
import akka.stream.scaladsl.Source
import org.apache.pekko.NotUsed
import org.apache.pekko.stream.scaladsl.Source
import com.advancedtelematic.data.DataType.ObjectStatus.ObjectStatus
import com.advancedtelematic.data.DataType.{ObjectId, ObjectStatus, TObject}
import com.advancedtelematic.libats.data.DataType.Namespace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import com.advancedtelematic.data.ClientDataType.StaticDelta
import com.advancedtelematic.data.DataType.{DeltaId, StaticDeltaMeta, SuperBlockHash}
import com.advancedtelematic.libats.data.DataType.Namespace
import com.advancedtelematic.libats.data.PaginationResult
import com.advancedtelematic.libats.data.PaginationResult.{Limit, Offset}
import com.advancedtelematic.libats.messaging_datatype.DataType.Commit
import com.advancedtelematic.libats.slick.codecs.SlickRefined.*
import com.advancedtelematic.libats.slick.db.SlickAnyVal.*
Expand Down Expand Up @@ -50,7 +51,7 @@ protected class StaticDeltaMetaRepository()(implicit db: Database, ec: Execution
.result
}

def findAll(ns: Namespace, status: StaticDeltaMeta.Status, offset: Long, limit: Long): Future[PaginationResult[StaticDelta]] = db.run {
def findAll(ns: Namespace, status: StaticDeltaMeta.Status, offset: Offset, limit: Limit): Future[PaginationResult[StaticDelta]] = db.run {
staticDeltas
.filter(_.namespace === ns)
.filter(_.status === status)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,23 +1,23 @@
package com.advancedtelematic.treehub.delta_store

import akka.http.scaladsl.model.HttpResponse
import akka.http.scaladsl.util.FastFuture
import akka.stream.scaladsl.Source
import akka.util.ByteString
import org.apache.pekko.http.scaladsl.model.HttpResponse
import org.apache.pekko.http.scaladsl.util.FastFuture
import org.apache.pekko.stream.scaladsl.Source
import org.apache.pekko.util.ByteString
import com.advancedtelematic.data.ClientDataType.{CommitInfo, CommitSize, StaticDelta}
import com.advancedtelematic.data.DataType.{DeltaId, DeltaIndexId, StaticDeltaIndex, StaticDeltaMeta, SuperBlockHash}
import com.advancedtelematic.libats.data.DataType.Namespace
import com.advancedtelematic.libats.data.PaginationResult
import com.advancedtelematic.libats.data.PaginationResult.{Limit, Offset}
import com.advancedtelematic.libats.messaging_datatype.DataType.Commit
import com.advancedtelematic.treehub.db.DbOps.PaginationResultOps
import com.advancedtelematic.treehub.db.DbOps.{PaginationResultLimitOps, PaginationResultOffsetOps}
import com.advancedtelematic.treehub.db.StaticDeltaMetaRepositorySupport
import com.advancedtelematic.treehub.http.Errors
import com.advancedtelematic.treehub.object_store.BlobStore
import org.slf4j.LoggerFactory
import slick.jdbc.MySQLProfile.api.*

import scala.async.Async.{async, await}
import scala.collection.{immutable, mutable}
import scala.concurrent.{ExecutionContext, Future}

class StaticDeltas(storage: BlobStore)(implicit val db: Database, ec: ExecutionContext) extends StaticDeltaMetaRepositorySupport {
Expand Down Expand Up @@ -63,7 +63,7 @@ class StaticDeltas(storage: BlobStore)(implicit val db: Database, ec: ExecutionC
}
}

def getAll(ns: Namespace, offset: Option[Long] = None, limit: Option[Long] = None): Future[PaginationResult[StaticDelta]] =
def getAll(ns: Namespace, offset: Option[Offset] = None, limit: Option[Limit] = None): Future[PaginationResult[StaticDelta]] =
staticDeltaMetaRepository.findAll(ns, StaticDeltaMeta.Status.Available, offset.orDefaultOffset, limit.orDefaultLimit)

def store(ns: Namespace, deltaId: DeltaId, path: String, data: Source[ByteString, ?], size: Long,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package com.advancedtelematic.treehub.http

class ConfResource {

import akka.http.scaladsl.server.Directives._
import org.apache.pekko.http.scaladsl.server.Directives._

val route = path("config") {

Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package com.advancedtelematic.treehub.http
import akka.http.scaladsl.model.*
import akka.http.scaladsl.model.headers.Location
import akka.http.scaladsl.server.*
import org.apache.pekko.http.scaladsl.model.*
import org.apache.pekko.http.scaladsl.model.headers.Location
import org.apache.pekko.http.scaladsl.server.*
import com.advancedtelematic.data.ClientDataType.CommitInfoRequest
import com.advancedtelematic.data.DataType.{CommitTupleOps, DeltaId, SuperBlockHash}
import com.advancedtelematic.data.GVariantEncoder.*
Expand All @@ -18,12 +18,13 @@ import io.circe.syntax.EncoderOps
import org.slf4j.LoggerFactory

import scala.util.Success
import com.advancedtelematic.libats.data.PaginationResult.*

class DeltaResource(namespace: Directive1[Namespace],
staticDeltas: StaticDeltas,
usageHandler: UsageMetricsRouter.HandlerRef) {

import akka.http.scaladsl.server.Directives.*
import org.apache.pekko.http.scaladsl.server.Directives.*

val _log = LoggerFactory.getLogger(this.getClass)

Expand Down Expand Up @@ -69,7 +70,7 @@ class DeltaResource(namespace: Directive1[Namespace],
}
} ~
post {
import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport._
implicit val commitKeyEncoder: KeyEncoder[Commit] = KeyEncoder.encodeKeyString.contramap(_.value)

entity(as[CommitInfoRequest]) { request =>
Expand All @@ -80,15 +81,14 @@ class DeltaResource(namespace: Directive1[Namespace],
(delete & path(PrefixedDeltaIdPath)) { id =>
val f = staticDeltas.markDeleted(ns, id)
complete(f.map(_ => StatusCodes.Accepted))
}
~
} ~
(pathEnd & parameters(Symbol("from").as[Commit], Symbol("to").as[Commit])) { (from, to) =>
val deltaId = (from, to).toDeltaId
val uri = Uri(s"/deltas/${deltaId.asPrefixedPath}")
complete(HttpResponse(StatusCodes.Found, headers = List(Location(uri))))
} ~
(pathEnd & parameters(Symbol("offset").as(nonNegativeLong).?, Symbol("limit").as(nonNegativeLong).?)) { (offset, limit) =>
val f = staticDeltas.getAll(ns, offset, limit).map(_.asJson.toString())
val f = staticDeltas.getAll(ns, offset.map(_.toOffset), limit.map(_.toLimit)).map(_.asJson.toString())
complete(f)
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.advancedtelematic.treehub.http

import akka.http.scaladsl.model.StatusCodes
import org.apache.pekko.http.scaladsl.model.StatusCodes
import com.advancedtelematic.data.DataType.{DeltaId, ObjectId, SuperBlockHash}
import com.advancedtelematic.libats.data.ErrorCode
import com.advancedtelematic.libats.http.Errors.{Error, RawError}
Expand Down
Loading
Loading