Started adding mongodb

build.gradle.kts

@@ -1,11 +1,14 @@
+val apache_commons_codec_version: String by project
 val ktor_version: String by project
 val kotlin_version: String by project
+val koin_version: String by project
 val logback_version: String by project
+val kmongo_version: String by project
 
 plugins {
     application
-    kotlin("jvm") version "1.7.0"
-    id("org.jetbrains.kotlin.plugin.serialization") version "1.7.0"
+    kotlin("jvm") version "1.7.10"
+    id("org.jetbrains.kotlin.plugin.serialization") version "1.7.10"
 }
 
 group = "dev.fyloz.backup"
@@ -23,12 +26,19 @@ repositories {
 }
 
 dependencies {
+    implementation("ch.qos.logback:logback-classic:$logback_version")
     implementation("io.ktor:ktor-server-core-jvm:$ktor_version")
     implementation("io.ktor:ktor-server-auth-jvm:$ktor_version")
     implementation("io.ktor:ktor-server-content-negotiation-jvm:$ktor_version")
+    implementation("io.ktor:ktor-server-call-logging:$ktor_version")
     implementation("io.ktor:ktor-serialization-kotlinx-json-jvm:$ktor_version")
     implementation("io.ktor:ktor-server-netty-jvm:$ktor_version")
-    implementation("ch.qos.logback:logback-classic:$logback_version")
+    implementation("io.ktor:ktor-server-status-pages:$ktor_version")
+    implementation("io.insert-koin:koin-ktor:$koin_version")
+    implementation("io.insert-koin:koin-logger-slf4j:$koin_version")
+    implementation("commons-codec:commons-codec:$apache_commons_codec_version")
+    implementation("org.litote.kmongo:kmongo:$kmongo_version")
+
     testImplementation("io.ktor:ktor-server-tests-jvm:$ktor_version")
     testImplementation("org.jetbrains.kotlin:kotlin-test-junit:$kotlin_version")
 }

docker-compose.yml

@@ -0,0 +1,7 @@
+version: "3"
+
+services:
+  backup.mongodb:
+    image: mongo:latest
+    ports:
+      - "27017:27017"

gradle.properties

@@ -1,4 +1,7 @@
-ktor_version=2.0.2
-kotlin_version=1.7.0
-logback_version=1.2.3
+apache_commons_codec_version=1.15
+ktor_version=2.0.3
+kotlin_version=1.7.10
+koin_version=3.2.0
+kmongo_version=4.7.0
+logback_version=1.2.11
 kotlin.code.style=official

src/main/kotlin/dev/fyloz/backup/ApiRouting.kt

@@ -1,51 +0,0 @@
-package dev.fyloz.backup
-
-import io.ktor.server.application.*
-import io.ktor.server.routing.*
-
-fun Application.configureApiRouting() {
-    routing {
-        route("/api/v1") {
-            configureAccountRoutes()
-        }
-    }
-}
-
-private fun Route.configureAccountRoutes() {
-    post("/login") {
-        // JWT tokens
-    }
-}
-
-private fun Route.configureBackupRoutes() {
-    route("/backup") {
-        post("create") {
-            // Start a new backup
-            // Returns a backup id and a public key to sign the data
-        }
-
-        post("finish/:id") {
-            // Finishes the backup with the given id
-            // Finished backups cannot be edited
-        }
-
-        put(":id") {
-            // Add a file to the backup with the given id
-            // The client has to send the file along with a checksums
-            // The file has to be encrypted with the public key created previously
-
-            // For incremental backups, the server will create a diff with the previous version
-        }
-
-        delete("cancel/:id") {
-            // Cancels the backup with the given id
-            // Removes all the files and denies the encryption key
-        }
-
-        delete("delete/:id") {
-            // Deletes the backup with the given id
-            // An incremental backup can only be deleted if it is the last one, as deleting previous ones will break the newest backups
-            // The last normal backup cannot be deleted as the incremental backups are be based on it
-        }
-    }
-}

src/main/kotlin/dev/fyloz/backup/Application.kt

@@ -1,16 +1,80 @@
 package dev.fyloz.backup
 
-import dev.fyloz.backup.plugins.configureSecurity
-import dev.fyloz.backup.plugins.configureSerialization
-import io.ktor.server.engine.*
+import dev.fyloz.backup.data.FileProvider
+import dev.fyloz.backup.data.LocalFileProvider
+import dev.fyloz.backup.data.MongoDatabase
+import dev.fyloz.backup.exceptions.HttpException
+import dev.fyloz.backup.exceptions.NotFoundException
+import dev.fyloz.backup.exceptions.ValidationException
+import dev.fyloz.backup.logic.injection.LogicInjection
+import dev.fyloz.backup.modules.backup.backupModule
+import dev.fyloz.backup.repositories.injection.RepositoryInjection
+import io.ktor.http.*
+import io.ktor.serialization.kotlinx.json.*
+import io.ktor.server.application.*
 import io.ktor.server.netty.*
+import io.ktor.server.plugins.callloging.*
+import io.ktor.server.plugins.contentnegotiation.*
+import io.ktor.server.plugins.statuspages.*
+import io.ktor.server.response.*
+import io.ktor.server.routing.*
+import kotlinx.serialization.SerializationException
+import org.koin.dsl.module
+import org.koin.ktor.plugin.Koin
+import org.koin.logger.slf4jLogger
+import org.slf4j.event.Level
 
-fun main() {
-    embeddedServer(Netty, port = 8080, host = "0.0.0.0") {
-        configureApiRouting()
-        configureSecurity()
-        configureSerialization()
-    }.start(wait = true)
+// https://github.com/mathias21/KtorEasy/tree/master
+
+fun main(args: Array<String>) = EngineMain.main(args)
+
+fun Application.module() {
+    install(CallLogging) {
+        level = Level.DEBUG
+    }
+    install(ContentNegotiation) {
+        json()
+    }
+
+    module {
+        install(Koin) {
+            slf4jLogger()
+            modules(
+                RepositoryInjection.koinBeans,
+                LogicInjection.koinBeans,
+
+                module {
+                    single { MongoDatabase() }
+                    single<FileProvider> { LocalFileProvider() }
+                }
+            )
+        }
+    }
+
+    install(Routing) {
+        route("/v1") {
+            backupModule()
+        }
+    }
+
+    install(StatusPages) {
+        exception<HttpException> { call, cause ->
+            val statusCode = when (cause) {
+                is ValidationException -> HttpStatusCode.BadRequest
+                is NotFoundException -> HttpStatusCode.NotFound
+            }
+
+            call.respondText(cause.message, status = statusCode)
+        }
+
+        exception<SerializationException> { call, cause ->
+            call.respondText(cause.localizedMessage, status = HttpStatusCode.BadRequest)
+        }
+
+        exception<Throwable> { call, cause ->
+            call.respondText(text = "500: $cause", status = HttpStatusCode.InternalServerError)
+        }
+    }
 }
 
 // Backups:

src/main/kotlin/dev/fyloz/backup/Constants.kt

@@ -0,0 +1,11 @@
+package dev.fyloz.backup
+
+object Constants {
+    object RequestParameters {
+        const val ID = "id"
+    }
+
+    object RequestHeaders {
+        const val CONTENT_MURMUR3 = "Content-Murmur3"
+    }
+}

src/main/kotlin/dev/fyloz/backup/data/FileProvider.kt

@@ -0,0 +1,14 @@
+package dev.fyloz.backup.data
+
+import java.nio.file.Files
+import kotlin.io.path.Path
+
+interface FileProvider {
+    fun save(data: ByteArray, path: String)
+}
+
+class LocalFileProvider : FileProvider {
+    override fun save(data: ByteArray, path: String) {
+        Files.write(Path(path), data)
+    }
+}

src/main/kotlin/dev/fyloz/backup/data/MongoDatabase.kt

@@ -0,0 +1,14 @@
+package dev.fyloz.backup.data
+
+import dev.fyloz.backup.entities.Backup
+import dev.fyloz.backup.entities.Machine
+import org.litote.kmongo.KMongo
+import org.litote.kmongo.getCollection
+
+class MongoDatabase {
+    private val client = KMongo.createClient()
+    private val database = client.getDatabase("backups")
+
+    val backupCollection = database.getCollection<Backup>()
+    val machineCollection = database.getCollection<Machine>()
+}

src/main/kotlin/dev/fyloz/backup/dtos/BackupResponses.kt

@@ -0,0 +1,20 @@
+package dev.fyloz.backup.dtos
+
+import dev.fyloz.backup.entities.BackupFile
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class CreateBackupResponse(
+    val id: String,
+    val machineId: String
+)
+
+@Serializable
+data class UploadFileResponse(
+    val id: String,
+    val name: String,
+    val path: String,
+    val murmur3Hash: String
+) {
+    constructor(file: BackupFile) : this(file.id.toString(), file.name, file.path, file.murmur3Hash)
+}

src/main/kotlin/dev/fyloz/backup/entities/Backup.kt

@@ -0,0 +1,21 @@
+package dev.fyloz.backup.entities
+
+import org.bson.codecs.pojo.annotations.BsonId
+import org.litote.kmongo.Id
+import java.time.LocalDateTime
+
+data class Backup(
+    @BsonId
+    val id: Id<Backup>,
+    val files: Collection<BackupFile> = setOf(),
+    val creationDateTime: LocalDateTime? = null
+)
+
+data class BackupFile(
+    @BsonId
+    val id: Id<BackupFile>,
+    val name: String,
+    val path: String,
+    val murmur3Hash: String
+)
+

src/main/kotlin/dev/fyloz/backup/entities/Machine.kt

@@ -0,0 +1,12 @@
+package dev.fyloz.backup.entities
+
+import org.bson.codecs.pojo.annotations.BsonId
+import org.litote.kmongo.Id
+import java.time.LocalDateTime
+
+data class Machine(
+    @BsonId
+    val id: Id<Machine>,
+    val backups: Collection<Backup>,
+    val creationDateTime: LocalDateTime
+)

src/main/kotlin/dev/fyloz/backup/exceptions/NotFoundException.kt

@@ -0,0 +1,9 @@
+package dev.fyloz.backup.exceptions
+
+sealed class HttpException(override val message: String) : RuntimeException(message)
+
+/** Exception thrown to indicate that a resource was not found. Will return HTTP 404 to the user. */
+class NotFoundException(message: String) : HttpException(message)
+
+/** Exception thrown to indicate that a request is not valid. Will return HTTP 400 to the user. */
+class ValidationException(message: String) : HttpException(message)

src/main/kotlin/dev/fyloz/backup/logic/BackupLogic.kt

@@ -0,0 +1,70 @@
+package dev.fyloz.backup.logic
+
+import dev.fyloz.backup.data.FileProvider
+import dev.fyloz.backup.dtos.CreateBackupResponse
+import dev.fyloz.backup.dtos.UploadFileResponse
+import dev.fyloz.backup.entities.Backup
+import dev.fyloz.backup.entities.BackupFile
+import dev.fyloz.backup.entities.Machine
+import dev.fyloz.backup.exceptions.NotFoundException
+import dev.fyloz.backup.exceptions.ValidationException
+import dev.fyloz.backup.modules.backup.AddBackupFileRequestBody
+import dev.fyloz.backup.repositories.BackupRepository
+import dev.fyloz.backup.repositories.MachineRepository
+import dev.fyloz.backup.utils.HashUtils
+import io.ktor.http.content.*
+import org.koin.core.component.KoinComponent
+import org.koin.core.component.inject
+import org.litote.kmongo.id.ObjectIdGenerator.generateNewId
+import org.litote.kmongo.id.StringId
+import java.time.LocalDateTime
+
+interface BackupLogic {
+    fun create(machineId: String): CreateBackupResponse
+    fun addFile(id: String, murmur3Hash: String, body: AddBackupFileRequestBody): UploadFileResponse
+}
+
+class DefaultBackupLogic : BackupLogic, KoinComponent {
+    private val repository by inject<BackupRepository>()
+    private val machineRepository by inject<MachineRepository>()
+    private val fileProvider by inject<FileProvider>()
+
+    override fun create(machineId: String): CreateBackupResponse {
+        val backup = Backup(generateNewId())
+
+        val machine = machineRepository.findById(machineId)
+        val updatedMachine = machine?.copy(backups = machine.backups + backup) ?: Machine(
+            StringId(machineId),
+            setOf(backup),
+            LocalDateTime.now()
+        )
+
+        repository.save(backup)
+        machineRepository.save(updatedMachine)
+
+        return CreateBackupResponse(backup.id.toString(), machineId)
+    }
+
+    override fun addFile(id: String, murmur3Hash: String, body: AddBackupFileRequestBody): UploadFileResponse {
+        val backup = repository.findById(id) ?: throw NotFoundException("Could not find a backup with the id '$id'")
+
+        saveFile(body.file, murmur3Hash)
+
+        val file = BackupFile(generateNewId(), body.originalName.value, body.path.value, murmur3Hash)
+
+        val updatedBackup = backup.copy(files = backup.files + file)
+        repository.save(updatedBackup)
+
+        return UploadFileResponse(file)
+    }
+
+    private fun saveFile(file: PartData.FileItem, murmur3Hash: String) {
+        val data = file.streamProvider().readAllBytes()
+        if (!HashUtils.murmur3Matches(data, murmur3Hash)) {
+            // File is corrupted, throw
+            throw ValidationException("Corrupted file, hash did not match")
+        }
+
+        fileProvider.save(data, "/home/william/Dev/Projects/backups/server/data/test.dat")
+    }
+}

src/main/kotlin/dev/fyloz/backup/logic/injection/LogicInjection.kt

@@ -0,0 +1,11 @@
+package dev.fyloz.backup.logic.injection
+
+import dev.fyloz.backup.logic.BackupLogic
+import dev.fyloz.backup.logic.DefaultBackupLogic
+import org.koin.dsl.module
+
+object LogicInjection {
+    val koinBeans = module {
+        single<BackupLogic> { DefaultBackupLogic() }
+    }
+}

src/main/kotlin/dev/fyloz/backup/modules/backup/AddBackupFileRequest.kt

@@ -0,0 +1,9 @@
+package dev.fyloz.backup.modules.backup
+
+import io.ktor.http.content.*
+
+class AddBackupFileRequestBody(map: Map<String?, PartData>) {
+    val file: PartData.FileItem by map
+    val originalName: PartData.FormItem by map
+    val path: PartData.FormItem by map
+}

src/main/kotlin/dev/fyloz/backup/modules/backup/BackupModule.kt

@@ -0,0 +1,67 @@
+package dev.fyloz.backup.modules.backup
+
+import dev.fyloz.backup.Constants
+import dev.fyloz.backup.logic.BackupLogic
+import dev.fyloz.backup.modules.backup.requests.CreateBackupRequest
+import dev.fyloz.backup.utils.validateAndGetHeader
+import io.ktor.http.content.*
+import io.ktor.server.application.*
+import io.ktor.server.request.*
+import io.ktor.server.response.*
+import io.ktor.server.routing.*
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+import org.koin.ktor.ext.inject
+
+fun Route.backupModule() {
+    val logic by inject<BackupLogic>()
+
+    route("/backup") {
+        post("create") {
+            val request = call.receive<CreateBackupRequest>()
+            val response = logic.create(request.machineId)
+
+            call.respond(response)
+
+            // Start a new backup
+            // Returns a backup id and a public key to sign the data
+
+            // https://stackoverflow.com/questions/40243857/how-to-encrypt-large-file-with-rsa
+        }
+
+        post("finish/:id") {
+            // Finishes the backup with the given id
+            // Finished backups cannot be edited
+        }
+
+        put("/{${Constants.RequestParameters.ID}}") {
+            val id = call.parameters[Constants.RequestParameters.ID]!!
+            val murmurHash3 = validateAndGetHeader(Constants.RequestHeaders.CONTENT_MURMUR3)
+
+            val multipartData = call.receiveMultipart().readAllParts()
+            val body = AddBackupFileRequestBody(multipartData.associateBy { it.name!! })
+
+            withContext(Dispatchers.IO) {
+                val backup = logic.addFile(id, murmurHash3, body)
+                call.respond(backup)
+            }
+
+            // Add a file to the backup with the given id
+            // The client has to send the file along with a checksums
+            // The file has to be encrypted with the public key created previously
+
+            // For incremental backups, the server will create a diff with the previous version
+        }
+
+        delete("cancel/:id") {
+            // Cancels the backup with the given id
+            // Removes all the files and revokes the encryption key
+        }
+
+        delete("delete/:id") {
+            // Deletes the backup with the given id
+            // An incremental backup can only be deleted if it is the last one, as deleting previous ones will break the newest backups
+            // The last normal backup cannot be deleted as the incremental backups are be based on it
+        }
+    }
+}

src/main/kotlin/dev/fyloz/backup/modules/backup/requests/CreateBackupRequest.kt

@@ -0,0 +1,8 @@
+package dev.fyloz.backup.modules.backup.requests
+
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class CreateBackupRequest(
+    val machineId: String
+)

src/main/kotlin/dev/fyloz/backup/plugins/Security.kt

@@ -1,12 +0,0 @@
-package dev.fyloz.backup.plugins
-
-import io.ktor.server.auth.*
-import io.ktor.util.*
-import io.ktor.server.application.*
-import io.ktor.server.response.*
-import io.ktor.server.request.*
-
-fun Application.configureSecurity() {
-
-
-}

src/main/kotlin/dev/fyloz/backup/plugins/Serialization.kt

@@ -1,20 +0,0 @@
-package dev.fyloz.backup.plugins
-
-import io.ktor.serialization.kotlinx.json.*
-import io.ktor.server.plugins.contentnegotiation.*
-import io.ktor.server.application.*
-import io.ktor.server.response.*
-import io.ktor.server.request.*
-import io.ktor.server.routing.*
-
-fun Application.configureSerialization() {
-    install(ContentNegotiation) {
-        json()
-    }
-
-    routing {
-        get("/json/kotlinx-serialization") {
-            call.respond(mapOf("hello" to "world"))
-        }
-    }
-}

src/main/kotlin/dev/fyloz/backup/repositories/BackupRepository.kt

@@ -0,0 +1,83 @@
+package dev.fyloz.backup.repositories
+
+import dev.fyloz.backup.data.MongoDatabase
+import dev.fyloz.backup.entities.Backup
+import org.bson.types.ObjectId
+import org.koin.core.component.KoinComponent
+import org.koin.core.component.inject
+import org.litote.kmongo.*
+import org.litote.kmongo.id.toId
+import kotlin.random.Random
+
+interface BackupRepository {
+    /** Returns all stored backups. */
+    fun findAll(): Collection<Backup>
+
+    /** Searches for a backup with the given [id] and returns it, if found. */
+    fun findById(id: String): Backup?
+
+    /** Saves the given [backup]. */
+    fun save(backup: Backup)
+
+    /** Deletes the backup with the given [id]. */
+    fun deleteById(id: String)
+}
+
+class MongoBackupRepository : BackupRepository, KoinComponent {
+    private val database by inject<MongoDatabase>()
+    private val backups = database.backupCollection
+
+    override fun findAll() = backups.find().toList()
+    override fun findById(id: String): Backup? {
+        val bsonId = ObjectId(id).toId<Backup>()
+        return backups.findOneById(bsonId)
+    }
+
+    override fun save(backup: Backup) {
+        backups.save(backup)
+    }
+
+    override fun deleteById(id: String) {
+        backups.deleteOneById(id)
+    }
+}
+
+class MemoryBackupRepository : BackupRepository {
+    private val backups = mutableMapOf<String, Backup>()
+    private val idCharPool = createIdCharPool()
+
+    override fun findAll() = backups.values
+    override fun findById(id: String) = backups[id]
+
+    // Creates a BSON id look-a-like
+    fun getNewId(): String {
+        var id = generateId()
+
+        while (id in backups) {
+            id = generateId()
+        }
+
+        return generateId()
+    }
+
+    private fun generateId() = (1..ID_STRING_LENGTH)
+        .map { Random.nextInt(idCharPool.size) }
+        .map { idCharPool[it] }
+        .joinToString("")
+
+    override fun save(backup: Backup) {
+        backups[backup.id.toString()] = backup
+    }
+
+    override fun deleteById(id: String) {
+        backups.remove(id)
+    }
+
+    companion object {
+        private const val ID_STRING_LENGTH = 16
+
+        fun createIdCharPool() = (0x41..0x5a)
+            .plus(0x61..0x7a)
+            .map { it.toChar() }
+    }
+}

src/main/kotlin/dev/fyloz/backup/repositories/MachineRepository.kt

@@ -0,0 +1,21 @@
+package dev.fyloz.backup.repositories
+
+import dev.fyloz.backup.entities.Machine
+
+interface MachineRepository {
+    /** Searches for the machine with the given [id] and returns it, if found. */
+    fun findById(id: String): Machine?
+
+    /** Saves the given [Machine]. */
+    fun save(machine: Machine)
+}
+
+class MemoryMachineRepository : MachineRepository {
+    private val machines = mutableMapOf<String, Machine>()
+
+    override fun findById(id: String) = machines[id]
+
+    override fun save(machine: Machine) {
+        machines[machine.id.toString()] = machine
+    }
+}

src/main/kotlin/dev/fyloz/backup/repositories/injection/RepositoryInjection.kt

@@ -0,0 +1,14 @@
+package dev.fyloz.backup.repositories.injection
+
+import dev.fyloz.backup.repositories.BackupRepository
+import dev.fyloz.backup.repositories.MachineRepository
+import dev.fyloz.backup.repositories.MemoryMachineRepository
+import dev.fyloz.backup.repositories.MongoBackupRepository
+import org.koin.dsl.module
+
+object RepositoryInjection {
+    val koinBeans = module {
+        single<BackupRepository> { MongoBackupRepository() }
+        single<MachineRepository> { MemoryMachineRepository() }
+    }
+}

src/main/kotlin/dev/fyloz/backup/utils/HashUtils.kt

@@ -0,0 +1,18 @@
+package dev.fyloz.backup.utils
+
+import org.apache.commons.codec.digest.MurmurHash3
+
+object HashUtils {
+    private const val murmurHash3Seed = 817344001
+
+    private fun murmur3(data: ByteArray): LongArray =
+        MurmurHash3.hash128x64(data, 0, data.size, murmurHash3Seed)
+
+    fun murmur3Matches(data: ByteArray, hash: String): Boolean {
+        val firstLong = hash.subSequence(0..(hash.length / 2)).toString().toLong()
+        val secondLong = hash.subSequence((hash.length / 2) + 1 until hash.length).toString().toLong()
+
+        val actualHash = murmur3(data)
+        return actualHash[0] == firstLong && actualHash[1] == secondLong
+    }
+}

src/main/kotlin/dev/fyloz/backup/utils/ValidationUtils.kt

@@ -0,0 +1,17 @@
+package dev.fyloz.backup.utils
+
+import dev.fyloz.backup.exceptions.ValidationException
+import io.ktor.server.application.*
+import io.ktor.util.pipeline.*
+
+/**
+ * Checks if a header with the [headerName] exists in the current request and returns its value, if present.
+ * Throws a [ValidationException] if the header has not been set.
+ */
+fun PipelineContext<Unit, ApplicationCall>.validateAndGetHeader(headerName: String): String {
+    if (headerName !in call.request.headers) {
+        throw ValidationException("Header '$headerName' is required")
+    }
+
+    return call.request.headers[headerName]!!
+}

src/main/resources/application.conf

@@ -0,0 +1,12 @@
+ktor {
+  deployment {
+    port = 8080
+    port = ${?PORT}
+  }
+
+  application {
+    modules = [ dev.fyloz.backup.ApplicationKt.module ]
+  }
+
+  development = true
+}

src/test/kotlin/dev/fyloz/backup/ApplicationTest.kt

@@ -1,21 +1,17 @@
 package dev.fyloz.backup
 
-import io.ktor.client.request.*
-import io.ktor.client.statement.*
-import io.ktor.http.*
 import io.ktor.server.testing.*
 import kotlin.test.Test
-import kotlin.test.assertEquals
 
 class ApplicationTest {
     @Test
     fun testRoot() = testApplication {
-        application {
-            configureApiRouting()
-        }
-        client.get("/").apply {
-            assertEquals(HttpStatusCode.OK, status)
-            assertEquals("Hello World!", bodyAsText())
-        }
+//        application {
+//            configureApiRouting()
+//        }
+//        client.get("/").apply {
+//            assertEquals(HttpStatusCode.OK, status)
+//            assertEquals("Hello World!", bodyAsText())
+//        }
     }
 }