feature/18-add-existing-files-cache #26

Merged
william merged 11 commits from feature/18-add-existing-files-cache into develop 2022-02-12 15:21:39 -05:00
10 changed files with 276 additions and 118 deletions
Showing only changes of commit e6b1ba3b45 - Show all commits

View File

@ -1,5 +1,6 @@
package dev.fyloz.colorrecipesexplorer.service
import dev.fyloz.colorrecipesexplorer.config.annotations.RequireDatabase
import dev.fyloz.colorrecipesexplorer.model.*
import dev.fyloz.colorrecipesexplorer.model.account.Group
import dev.fyloz.colorrecipesexplorer.model.validation.or
@ -9,10 +10,8 @@ import dev.fyloz.colorrecipesexplorer.service.files.WriteableFileService
import dev.fyloz.colorrecipesexplorer.service.users.GroupService
import dev.fyloz.colorrecipesexplorer.utils.setAll
import org.springframework.context.annotation.Lazy
import org.springframework.context.annotation.Profile
import org.springframework.stereotype.Service
import org.springframework.web.multipart.MultipartFile
import java.io.File
import java.time.LocalDate
import java.time.Period
import javax.transaction.Transactional
@ -45,7 +44,7 @@ interface RecipeService :
}
@Service
@Profile("!emergency")
@RequireDatabase
class RecipeServiceImpl(
recipeRepository: RecipeRepository,
val companyService: CompanyService,
@ -213,29 +212,20 @@ interface RecipeImageService {
/** Deletes the image with the given [name] for the given [recipe]. */
fun delete(recipe: Recipe, name: String)
/** Gets the directory containing all images of the given [Recipe]. */
fun Recipe.getDirectory(): File
}
const val RECIPE_IMAGE_ID_DELIMITER = "_"
const val RECIPE_IMAGE_EXTENSION = ".jpg"
@Service
@Profile("!emergency")
@RequireDatabase
class RecipeImageServiceImpl(
val fileService: WriteableFileService
) : RecipeImageService {
override fun getAllImages(recipe: Recipe): Set<String> {
val recipeDirectory = recipe.getDirectory()
if (!recipeDirectory.exists() || !recipeDirectory.isDirectory) {
return setOf()
}
return recipeDirectory.listFiles()!! // Should never be null because we check if recipeDirectory exists and is a directory before
.filterNotNull()
override fun getAllImages(recipe: Recipe) =
fileService.listDirectoryFiles(recipe.imagesDirectoryPath)
.map { it.name }
.toSet()
}
override fun download(image: MultipartFile, recipe: Recipe): String {
/** Gets the next id available for a new image for the given [recipe]. */
@ -252,17 +242,15 @@ class RecipeImageServiceImpl(
} + 1L
}
return getImageFileName(recipe, getNextAvailableId()).apply {
fileService.write(image, getImagePath(recipe, this), true)
return getImageFileName(recipe, getNextAvailableId()).also {
with(getImagePath(recipe, it)) {
fileService.writeToDirectory(image, this, recipe.imagesDirectoryPath, true)
}
}
}
override fun delete(recipe: Recipe, name: String) =
fileService.delete(getImagePath(recipe, name))
override fun Recipe.getDirectory(): File = File(with(fileService) {
this@getDirectory.imagesDirectoryPath.fullPath().path
})
fileService.deleteFromDirectory(getImagePath(recipe, name), recipe.imagesDirectoryPath)
private fun getImageFileName(recipe: Recipe, id: Long) =
"${recipe.name}$RECIPE_IMAGE_ID_DELIMITER$id"

View File

@ -0,0 +1,150 @@
package dev.fyloz.colorrecipesexplorer.service.files
import dev.fyloz.colorrecipesexplorer.JavaFile
import dev.fyloz.colorrecipesexplorer.utils.File
import dev.fyloz.colorrecipesexplorer.utils.FilePath
import mu.KotlinLogging
object FileCache {
private val logger = KotlinLogging.logger {}
private val cache = hashMapOf<String, CachedFileSystemItem>()
operator fun contains(filePath: FilePath) =
filePath.path in cache
operator fun get(filePath: FilePath) =
cache[filePath.path]
fun getDirectory(filePath: FilePath) =
if (directoryExists(filePath)) {
this[filePath] as CachedDirectory
} else {
null
}
fun getFile(filePath: FilePath) =
if (fileExists(filePath)) {
this[filePath] as CachedFile
} else {
null
}
private operator fun set(filePath: FilePath, item: CachedFileSystemItem) {
cache[filePath.path] = item
}
fun exists(filePath: FilePath) =
filePath in this && cache[filePath.path]!!.exists
fun directoryExists(filePath: FilePath) =
exists(filePath) && this[filePath] is CachedDirectory
fun fileExists(filePath: FilePath) =
exists(filePath) && this[filePath] is CachedFile
fun setExists(filePath: FilePath, exists: Boolean = true) {
if (filePath !in this) {
load(filePath)
}
this[filePath] = this[filePath]!!.clone(exists)
logger.debug("Updated FileCache state: ${filePath.path} exists -> $exists")
}
fun setDoesNotExists(filePath: FilePath) =
setExists(filePath, false)
fun load(filePath: FilePath) =
with(JavaFile(filePath.path).toFileSystemItem()) {
cache[filePath.path] = this
logger.debug("Loaded file at ${filePath.path} into FileCache")
}
fun addContent(filePath: FilePath, childFilePath: FilePath) {
val directory = prepareDirectory(filePath) ?: return
val updatedContent = setOf(
*directory.content.toTypedArray(),
JavaFile(childFilePath.path).toFileSystemItem()
)
this[filePath] = directory.copy(content = updatedContent)
logger.debug("Added child ${childFilePath.path} to ${filePath.path} in FileCache")
}
fun removeContent(filePath: FilePath, childFilePath: FilePath) {
val directory = prepareDirectory(filePath) ?: return
val updatedContent = directory.content
.filter { it.path.path != childFilePath.path }
.toSet()
this[filePath] = directory.copy(content = updatedContent)
logger.debug("Removed child ${childFilePath.path} from ${filePath.path} in FileCache")
}
private fun prepareDirectory(filePath: FilePath): CachedDirectory? {
if (!directoryExists(filePath)) {
logger.warn("Cannot add child to ${filePath.path} because it is not in the cache")
return null
}
val directory = getDirectory(filePath)
if (directory == null) {
logger.warn("Cannot add child to ${filePath.path} because it is not a directory")
return null
}
return directory
}
}
interface CachedFileSystemItem {
val name: String
val path: FilePath
val exists: Boolean
fun clone(exists: Boolean): CachedFileSystemItem
}
data class CachedFile(
override val name: String,
override val path: FilePath,
override val exists: Boolean
) : CachedFileSystemItem {
constructor(file: File) : this(file.name, file.toFilePath(), file.exists() && file.isFile)
override fun clone(exists: Boolean) =
this.copy(exists = exists)
}
data class CachedDirectory(
override val name: String,
override val path: FilePath,
override val exists: Boolean,
val content: Set<CachedFileSystemItem> = setOf()
) : CachedFileSystemItem {
constructor(file: File) : this(file.name, file.toFilePath(), file.exists() && file.isDirectory, file.fetchContent())
val contentFiles: Collection<CachedFile>
get() = content.filterIsInstance<CachedFile>()
override fun clone(exists: Boolean) =
this.copy(exists = exists)
companion object {
private fun File.fetchContent() =
(this.file.listFiles() ?: arrayOf<JavaFile>())
.filterNotNull()
.map { it.toFileSystemItem() }
.toSet()
}
}
fun JavaFile.toFileSystemItem() =
if (this.isDirectory) {
CachedDirectory(File(this))
} else {
CachedFile(File(this))
}

View File

@ -1,32 +0,0 @@
package dev.fyloz.colorrecipesexplorer.service.files
import dev.fyloz.colorrecipesexplorer.utils.FilePath
import mu.KotlinLogging
object FileExistCache {
private val logger = KotlinLogging.logger {}
private val map = hashMapOf<String, Boolean>()
/** Checks if the given [path] is in the cache. */
operator fun contains(path: FilePath) =
path.path in map
/** Checks if the file at the given [path] exists. */
fun exists(path: FilePath) =
map[path.path] ?: false
/** Sets the file at the given [path] as existing. */
fun setExists(path: FilePath) =
set(path, true)
/** Sets the file at the given [path] as not existing. */
fun setDoesNotExists(path: FilePath) =
set(path, false)
/** Sets if the file at the given [path] [exists]. */
fun set(path: FilePath, exists: Boolean) {
map[path.path] = exists
logger.debug("Updated FileExistCache state: ${path.path} -> $exists")
}
}

View File

@ -28,8 +28,11 @@ interface FileService {
/** Reads the file at the given [path]. */
fun read(path: String): Resource
/** List the files contained in the folder at the given [path]. Returns an empty collection if the directory does not exist. */
fun listDirectoryFiles(path: String): Collection<CachedFile>
/** Completes the path of the given [String] by adding the working directory. */
fun String.fullPath(): FilePath
fun fullPath(path: String): FilePath
}
interface WriteableFileService : FileService {
@ -42,8 +45,14 @@ interface WriteableFileService : FileService {
/** Writes the given [data] to the given [path]. If the file at the path already exists, it will be overwritten if [overwrite] is enabled. */
fun write(data: ByteArrayResource, path: String, overwrite: Boolean)
/** Writes the given [data] to the given [path], and specify the [parentPath]. If the file at the path already exists, it will be overwritten if [overwrite] is enabled. */
fun writeToDirectory(data: MultipartFile, path: String, parentPath: String, overwrite: Boolean)
/** Deletes the file at the given [path]. */
fun delete(path: String)
/** Deletes the file at the given [path], and specify the [parentPath]. */
fun deleteFromDirectory(path: String, parentPath: String)
}
@Service
@ -53,20 +62,20 @@ class FileServiceImpl(
private val logger = KotlinLogging.logger {}
override fun exists(path: String): Boolean {
val fullPath = path.fullPath()
return if (fullPath in FileExistCache) {
FileExistCache.exists(fullPath)
val fullPath = fullPath(path)
return if (fullPath in FileCache) {
FileCache.exists(fullPath)
} else {
withFileAt(fullPath) {
(this.exists() && this.isFile).also {
FileExistCache.set(fullPath, it)
FileCache.setExists(fullPath, it)
}
}
}
}
override fun read(path: String) = ByteArrayResource(
withFileAt(path.fullPath()) {
withFileAt(fullPath(path)) {
if (!exists(path)) throw FileNotFoundException(path)
try {
readBytes()
@ -76,13 +85,23 @@ class FileServiceImpl(
}
)
override fun listDirectoryFiles(path: String): Collection<CachedFile> =
with(fullPath(path)) {
if (this !in FileCache) {
FileCache.load(this)
}
(FileCache.getDirectory(this) ?: return setOf())
.contentFiles
}
override fun create(path: String) {
val fullPath = path.fullPath()
val fullPath = fullPath(path)
if (!exists(path)) {
try {
withFileAt(fullPath) {
this.create()
FileExistCache.setExists(fullPath)
FileCache.setExists(fullPath)
logger.info("Created file at '${fullPath.path}'")
}
@ -104,14 +123,19 @@ class FileServiceImpl(
this.writeBytes(data.byteArray)
}
override fun writeToDirectory(data: MultipartFile, path: String, parentPath: String, overwrite: Boolean) {
FileCache.addContent(fullPath(parentPath), fullPath(path))
write(data, path, overwrite)
}
override fun delete(path: String) {
try {
val fullPath = path.fullPath()
val fullPath = fullPath(path)
withFileAt(fullPath) {
if (!exists(path)) throw FileNotFoundException(path)
this.delete()
FileExistCache.setDoesNotExists(fullPath)
FileCache.setDoesNotExists(fullPath)
logger.info("Deleted file at '${fullPath.path}'")
}
@ -120,16 +144,21 @@ class FileServiceImpl(
}
}
override fun String.fullPath(): FilePath {
BANNED_FILE_PATH_SHARDS
.firstOrNull { this.contains(it) }
?.let { throw InvalidFilePathException(this, it) }
override fun deleteFromDirectory(path: String, parentPath: String) {
FileCache.removeContent(fullPath(parentPath), fullPath(path))
delete(path)
}
return FilePath("${creProperties.dataDirectory}/$this")
override fun fullPath(path: String): FilePath {
BANNED_FILE_PATH_SHARDS
.firstOrNull { path.contains(it) }
?.let { throw InvalidFilePathException(path, it) }
return FilePath("${creProperties.dataDirectory}/$path")
}
private fun prepareWrite(path: String, overwrite: Boolean, op: File.() -> Unit) {
val fullPath = path.fullPath()
val fullPath = fullPath(path)
if (exists(path)) {
if (!overwrite) throw FileExistsException(path)

View File

@ -10,17 +10,26 @@ class ResourceFileService(
private val resourceLoader: ResourceLoader
) : FileService {
override fun exists(path: String) =
path.fullPath().resource.exists()
fullPath(path).resource.exists()
override fun read(path: String): Resource =
path.fullPath().resource.also {
fullPath(path).resource.also {
if (!it.exists()) {
throw FileNotFoundException(path)
}
}
override fun String.fullPath() =
FilePath("classpath:${this}")
override fun listDirectoryFiles(path: String): Collection<CachedFile> {
val content = fullPath(path).resource.file.listFiles() ?: return setOf()
return content
.filterNotNull()
.filter { it.isFile }
.map { it.toFileSystemItem() as CachedFile }
}
override fun fullPath(path: String) =
FilePath("classpath:${path}")
val FilePath.resource: Resource
get() = resourceLoader.getResource(this.path)

View File

@ -6,12 +6,21 @@ import java.nio.file.Path
/** Mockable file wrapper, to prevent issues when mocking [java.io.File]. */
class File(val file: JavaFile) {
val name: String
get() = file.name
val isFile: Boolean
get() = file.isFile
val isDirectory: Boolean
get() = file.isDirectory
fun toPath(): Path =
file.toPath()
fun toFilePath(): FilePath =
FilePath(file.path)
fun exists() =
file.exists()

View File

@ -8,9 +8,9 @@
%green(%d{ISO8601}) %highlight(%-5level) [%blue(%t)] %yellow(%C{36}): %msg%n%throwable
</Pattern>
</layout>
<!-- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">-->
<!-- <level>INFO</level>-->
<!-- </filter>-->
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
</appender>
<appender name="LOG_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">

View File

@ -6,8 +6,10 @@ import dev.fyloz.colorrecipesexplorer.model.*
import dev.fyloz.colorrecipesexplorer.model.account.group
import dev.fyloz.colorrecipesexplorer.repository.RecipeRepository
import dev.fyloz.colorrecipesexplorer.service.config.ConfigurationService
import dev.fyloz.colorrecipesexplorer.service.files.CachedFile
import dev.fyloz.colorrecipesexplorer.service.files.WriteableFileService
import dev.fyloz.colorrecipesexplorer.service.users.GroupService
import dev.fyloz.colorrecipesexplorer.utils.FilePath
import io.mockk.*
import org.junit.jupiter.api.AfterEach
import org.junit.jupiter.api.Test
@ -15,7 +17,6 @@ import org.junit.jupiter.api.TestInstance
import org.junit.jupiter.api.assertThrows
import org.springframework.mock.web.MockMultipartFile
import org.springframework.web.multipart.MultipartFile
import java.io.File
import java.time.LocalDate
import java.time.Period
import kotlin.test.*
@ -30,7 +31,17 @@ class RecipeServiceTest :
private val recipeStepService: RecipeStepService = mock()
private val configService: ConfigurationService = mock()
override val service: RecipeService =
spy(RecipeServiceImpl(repository, companyService, mixService, recipeStepService, groupService, mock(), configService))
spy(
RecipeServiceImpl(
repository,
companyService,
mixService,
recipeStepService,
groupService,
mock(),
configService
)
)
private val company: Company = company(id = 0L)
override val entity: Recipe = recipe(id = 0L, name = "recipe", company = company)
@ -273,18 +284,7 @@ private class RecipeImageServiceTestContext {
val recipe = spyk(recipe())
val recipeImagesIds = setOf(1L, 10L, 21L)
val recipeImagesNames = recipeImagesIds.map { it.imageName }.toSet()
val recipeImagesFiles = recipeImagesNames.map { File(it) }.toTypedArray()
val recipeDirectory = mockk<File> {
every { exists() } returns true
every { isDirectory } returns true
every { listFiles() } returns recipeImagesFiles
}
init {
with(recipeImageService) {
every { recipe.getDirectory() } returns recipeDirectory
}
}
val recipeImagesFiles = recipeImagesNames.map { CachedFile(it, FilePath(it), true) }
val Long.imageName
get() = "${recipe.name}$RECIPE_IMAGE_ID_DELIMITER$this"
@ -308,6 +308,8 @@ class RecipeImageServiceTest {
@Test
fun `getAllImages() returns a Set containing the name of every files in the recipe's directory`() {
test {
every { fileService.listDirectoryFiles(any()) } returns recipeImagesFiles
val foundImagesNames = recipeImageService.getAllImages(recipe)
assertEquals(recipeImagesNames, foundImagesNames)
@ -317,7 +319,7 @@ class RecipeImageServiceTest {
@Test
fun `getAllImages() returns an empty Set when the recipe's directory does not exists`() {
test {
every { recipeDirectory.exists() } returns false
every { fileService.listDirectoryFiles(any()) } returns emptySet()
assertTrue {
recipeImageService.getAllImages(recipe).isEmpty()
@ -335,12 +337,15 @@ class RecipeImageServiceTest {
val expectedImageName = expectedImageId.imageName
val expectedImagePath = expectedImageName.imagePath
every { fileService.listDirectoryFiles(any()) } returns recipeImagesFiles
every { fileService.writeToDirectory(any(), any(), any(), any()) } just runs
val foundImageName = recipeImageService.download(mockImage, recipe)
assertEquals(expectedImageName, foundImageName)
verify {
fileService.write(mockImage, expectedImagePath, true)
fileService.writeToDirectory(mockImage, expectedImagePath, any(), true)
}
}
}
@ -353,10 +358,12 @@ class RecipeImageServiceTest {
val imageName = recipeImagesIds.first().imageName
val imagePath = imageName.imagePath
every { fileService.deleteFromDirectory(any(), any()) } just runs
recipeImageService.delete(recipe, imageName)
verify {
fileService.delete(imagePath)
fileService.deleteFromDirectory(imagePath, any())
}
}
}

View File

@ -43,8 +43,8 @@ class FileServiceTest {
}
private fun whenFileCached(cached: Boolean = true, test: () -> Unit) {
mockkObject(FileExistCache) {
every { FileExistCache.contains(any()) } returns cached
mockkObject(FileCache) {
every { FileCache.contains(any()) } returns cached
test()
}
@ -106,34 +106,34 @@ class FileServiceTest {
@Test
fun `exists() returns true when the file at the given path is cached as existing`() {
whenFileCached {
every { FileExistCache.exists(any()) } returns true
every { FileCache.exists(any()) } returns true
assertTrue { fileService.exists(mockFilePath) }
verify {
FileExistCache.contains(any())
FileExistCache.exists(any())
FileCache.contains(any())
FileCache.exists(any())
mockFile wasNot called
}
confirmVerified(FileExistCache, mockFile)
confirmVerified(FileCache, mockFile)
}
}
@Test
fun `exists() returns false when the file at the given path is cached as not existing`() {
whenFileCached {
every { FileExistCache.exists(any()) } returns false
every { FileCache.exists(any()) } returns false
assertFalse { fileService.exists(mockFilePath) }
verify {
FileExistCache.contains(any())
FileExistCache.exists(any())
FileCache.contains(any())
FileCache.exists(any())
mockFile wasNot called
}
confirmVerified(FileExistCache, mockFile)
confirmVerified(FileCache, mockFile)
}
}
@ -180,16 +180,16 @@ class FileServiceTest {
whenFileNotCached {
mockkStatic(File::create) {
every { mockFile.create() } just Runs
every { FileExistCache.setExists(any()) } just Runs
every { FileCache.setExists(any()) } just Runs
fileService.create(mockFilePath)
verify {
mockFile.create()
FileExistCache.setExists(any())
FileCache.setExists(any())
}
confirmVerified(mockFile, FileExistCache)
confirmVerified(mockFile, FileCache)
}
}
}
@ -278,16 +278,16 @@ class FileServiceTest {
whenMockFilePathExists {
whenFileCached {
every { mockFile.delete() } returns true
every { FileExistCache.setDoesNotExists(any()) } just Runs
every { FileCache.setDoesNotExists(any()) } just Runs
fileService.delete(mockFilePath)
verify {
mockFile.delete()
FileExistCache.setDoesNotExists(any())
FileCache.setDoesNotExists(any())
}
confirmVerified(mockFile, FileExistCache)
confirmVerified(mockFile, FileCache)
}
}
}
@ -317,7 +317,7 @@ class FileServiceTest {
@Test
fun `fullPath() appends the given path to the given working directory`() {
with(fileService) {
val fullFilePath = mockFilePath.fullPath()
val fullFilePath = fullPath(mockFilePath)
assertEquals("${creProperties.dataDirectory}/$mockFilePath", fullFilePath.path)
}
@ -329,7 +329,7 @@ class FileServiceTest {
BANNED_FILE_PATH_SHARDS.forEach {
val maliciousPath = "$it/$mockFilePath"
with(assertThrows<InvalidFilePathException> { maliciousPath.fullPath() }) {
with(assertThrows<InvalidFilePathException> { fullPath(maliciousPath) }) {
assertEquals(maliciousPath, this.path)
assertEquals(it, this.fragment)
}

View File

@ -27,7 +27,7 @@ class ResourceFileServiceTest {
private fun existsTest(shouldExists: Boolean, test: (String) -> Unit) {
val path = "unit_test_resource"
with(service) {
every { path.fullPath() } returns mockk {
every { fullPath(path) } returns mockk {
every { resource } returns mockk {
every { exists() } returns shouldExists
}
@ -61,7 +61,7 @@ class ResourceFileServiceTest {
}
val path = "unit_test_path"
with(service) {
every { path.fullPath() } returns mockk {
every { fullPath(path) } returns mockk {
every { resource } returns mockResource
}
@ -92,11 +92,9 @@ class ResourceFileServiceTest {
val path = "unit_test_path"
val expectedPath = "classpath:$path"
with(service) {
val found = path.fullPath()
val found = service.fullPath(path)
assertEquals(expectedPath, found.path)
}
assertEquals(expectedPath, found.path)
}
@Test