From 086a68d86ab65e4eb20f96a99f853291c84a4331 Mon Sep 17 00:00:00 2001 From: batalin Date: Tue, 29 Oct 2019 17:35:28 +0200 Subject: [PATCH 01/14] Just for code review (BULK). This code contains error. --- .../offer/PostgresOfferRepositoryImpl.kt | 62 ++++++++++++++++++- .../bitclave/node/services/v1/OfferService.kt | 6 +- 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt b/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt index d06735ff..cd7a227f 100644 --- a/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt +++ b/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt @@ -21,6 +21,7 @@ import org.springframework.data.domain.SliceImpl import org.springframework.data.repository.findByIdOrNull import org.springframework.stereotype.Component import java.math.BigInteger +import java.text.SimpleDateFormat import java.util.HashMap import javax.persistence.EntityManager import kotlin.system.measureTimeMillis @@ -31,7 +32,8 @@ class PostgresOfferRepositoryImpl( val repository: OfferCrudRepository, val offerSearchRepository: OfferSearchCrudRepository, val entityManager: EntityManager, - val wsService: WsService + val wsService: WsService, + val em: EntityManager ) : OfferRepository { override fun saveOffer(offer: Offer): Offer { @@ -46,7 +48,61 @@ class PostgresOfferRepositoryImpl( } override fun saveAll(offers: List): List { - val result = repository.saveAll(offers).toList() + // native spring implementation + // val result = repository.saveAll(offers).toList() + // return syncElementCollections(result) + + var result: List = listOf() + val offersForInserting = offers.filter { it.id == 0L } + + val insertedOffersValues = offersForInserting.map { + val isoPattern = "yyyy-MM-dd'T'HH:mm:ss'Z'" + val updatedAt = SimpleDateFormat(isoPattern).format(it.updatedAt) + val createdAt = SimpleDateFormat(isoPattern).format(it.createdAt) + "(nextval('offer_id_seq')," + + " '${it.title}', '${it.description}', '${it.owner}', " + + "'${it.imageUrl}', '${it.worth}', '$createdAt', '$updatedAt')" + } + if (insertedOffersValues.isNotEmpty()) { + val springQueryTiming = measureTimeMillis { + // create offers + val insertOffers = "INSERT INTO offer " + + "(id, title, description, owner, image_url, worth, created_at, updated_at) VALUES \n" + val insertOffersQuery = insertOffers + insertedOffersValues.joinToString(",\n") + + "\n RETURNING id;" + val createdOfferIds: List = em.createNativeQuery(insertOffersQuery).resultList as List +// println("result: $createdOfferIds") + + // create tags + val insertTags = "INSERT INTO offer_tags (offer_id, tags, tags_key) VALUES \n" + val insertedOfferTagsValues = offersForInserting.mapIndexed { index, offer -> + offer.tags.map { "( ${createdOfferIds[index]}, '${it.value}', '${it.key}' )" } + }.flatten().joinToString(",\n") + val insertOfferTagsQuery = insertTags + insertedOfferTagsValues + em.createNativeQuery(insertOfferTagsQuery).executeUpdate() + + // compare + val insertCompare = "INSERT INTO offer_compare (offer_id, compare, compare_key) VALUES \n" + val insertedOfferCompareValues = offersForInserting.mapIndexed { index, offer -> + offer.compare.map { "( ${createdOfferIds[index]}, '${it.value}', '${it.key}' )" } + }.flatten().joinToString(",\n") + val insertOfferCompareQuery = insertCompare + insertedOfferCompareValues + em.createNativeQuery(insertOfferCompareQuery).executeUpdate() + + // rules + val insertRules = "INSERT INTO offer_rules (offer_id, rules, rules_key) VALUES \n" + val insertedOfferRulesValues = offersForInserting.mapIndexed { index, offer -> + offer.rules.map { "( ${createdOfferIds[index]}, ${it.value.ordinal}, '${it.key}' )" } + }.flatten().joinToString(",\n") + val insertOfferRulesQuery = insertRules + insertedOfferRulesValues + em.createNativeQuery(insertOfferRulesQuery).executeUpdate() + + val ids = createdOfferIds.joinToString(", ") + val query = "SELECT * FROM offer WHERE offer.id IN ($ids)" + result = em.createNativeQuery(query, Offer::class.java).resultList as List + } + println(" -- save all (native query) timing $springQueryTiming") + } return syncElementCollections(result) } @@ -68,7 +124,7 @@ class PostgresOfferRepositoryImpl( override fun deleteOffers(owner: String): Int { val deletedOffers = repository.deleteByOwner(owner) deletedOffers.forEach { - wsService.sendEvent(OfferEvent.OnDelete, it.id) +// wsService.sendEvent(OfferEvent.OnDelete, it.id) } return deletedOffers.size diff --git a/src/main/kotlin/com/bitclave/node/services/v1/OfferService.kt b/src/main/kotlin/com/bitclave/node/services/v1/OfferService.kt index 9b820632..c4a79be1 100644 --- a/src/main/kotlin/com/bitclave/node/services/v1/OfferService.kt +++ b/src/main/kotlin/com/bitclave/node/services/v1/OfferService.kt @@ -141,7 +141,7 @@ class OfferService( val saveAllTiming = measureTimeMillis { result = offerRepository.changeStrategy(strategy).saveAll(readyForSaveOffers) } -// logger.debug(" - save all timing $saveAllTiming") + Logger.debug(" - save all timing $saveAllTiming") val prices = result.mapIndexed { index, offer -> readyForSaveOffers[index].offerPrices.map { offerPrice -> @@ -154,7 +154,7 @@ class OfferService( val saveAllPricesTiming = measureTimeMillis { offerPriceRepository.changeStrategy(strategy).saveAllPrices(prices) } -// logger.debug(" - save all prices timing $saveAllPricesTiming") + Logger.debug(" - save all prices timing $saveAllPricesTiming") val offersIdsForCleanupOfferSearches = offers.filter { it.id != 0L }.map { it.id } val deleteOfferSearchTiming = measureTimeMillis { @@ -162,7 +162,7 @@ class OfferService( offerSearchService.deleteByOfferIds(offersIdsForCleanupOfferSearches, strategy) } } -// logger.debug(" - delete all offer searches by offerId timing $deleteOfferSearchTiming") + Logger.debug(" - delete all offer searches by offerId timing $deleteOfferSearchTiming") result.map { it.id } }) From f54d21dc0a9eb70c427c43bb3d26784e3b01c47a Mon Sep 17 00:00:00 2001 From: batalin Date: Wed, 30 Oct 2019 17:39:49 +0200 Subject: [PATCH 02/14] update code ready for testing --- .../dev/VerifyConsistencyController.kt | 2 +- .../offer/PostgresOfferRepositoryImpl.kt | 50 +++++++++++++------ 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/src/main/kotlin/com/bitclave/node/controllers/dev/VerifyConsistencyController.kt b/src/main/kotlin/com/bitclave/node/controllers/dev/VerifyConsistencyController.kt index 6efbe0e7..65bcba71 100644 --- a/src/main/kotlin/com/bitclave/node/controllers/dev/VerifyConsistencyController.kt +++ b/src/main/kotlin/com/bitclave/node/controllers/dev/VerifyConsistencyController.kt @@ -455,7 +455,7 @@ class VerifyConsistencyController( .thenCompose { offerSearchService.getOfferInteractionsByOfferIdsAndOwners( request.data!!.offerIds, - request.data!!.owners, + request.data.owners, getStrategyType(strategy) ) }.exceptionally { e -> diff --git a/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt b/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt index cd7a227f..d6871cf5 100644 --- a/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt +++ b/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt @@ -24,6 +24,7 @@ import java.math.BigInteger import java.text.SimpleDateFormat import java.util.HashMap import javax.persistence.EntityManager +import org.springframework.transaction.annotation.Transactional import kotlin.system.measureTimeMillis @Component @@ -32,8 +33,7 @@ class PostgresOfferRepositoryImpl( val repository: OfferCrudRepository, val offerSearchRepository: OfferSearchCrudRepository, val entityManager: EntityManager, - val wsService: WsService, - val em: EntityManager + val wsService: WsService ) : OfferRepository { override fun saveOffer(offer: Offer): Offer { @@ -46,21 +46,34 @@ class PostgresOfferRepositoryImpl( return result } - + @Transactional override fun saveAll(offers: List): List { // native spring implementation // val result = repository.saveAll(offers).toList() // return syncElementCollections(result) var result: List = listOf() - val offersForInserting = offers.filter { it.id == 0L } + // prepare for delete + val offerForDeletion = offers.filter { it.id != 0L } + if (offerForDeletion.isNotEmpty()) { + val ids = offerForDeletion.joinToString(", ") + var query = "DELETE FROM offer where offer.id IN ($ids)" + entityManager.createNativeQuery(query) .executeUpdate() + query = "DELETE FROM offer_tags where offer_tags.offer_id IN ($ids)" + entityManager.createNativeQuery(query).executeUpdate() + query = "DELETE FROM offer_compare where offer_compare.offer.id IN ($ids)" + entityManager.createNativeQuery(query).executeUpdate() + query = "DELETE FROM offer_rules where offer_rules.offer.id IN ($ids)" + entityManager.createNativeQuery(query).executeUpdate() + } - val insertedOffersValues = offersForInserting.map { + val insertedOffersValues = offers.map { val isoPattern = "yyyy-MM-dd'T'HH:mm:ss'Z'" val updatedAt = SimpleDateFormat(isoPattern).format(it.updatedAt) val createdAt = SimpleDateFormat(isoPattern).format(it.createdAt) - "(nextval('offer_id_seq')," + - " '${it.title}', '${it.description}', '${it.owner}', " + + val id = if (it.id == 0L) "(nextval('offer_id_seq')," else it.id.toString() + + "$id, '${it.title}', '${it.description}', '${it.owner}', " + "'${it.imageUrl}', '${it.worth}', '$createdAt', '$updatedAt')" } if (insertedOffersValues.isNotEmpty()) { @@ -70,36 +83,41 @@ class PostgresOfferRepositoryImpl( "(id, title, description, owner, image_url, worth, created_at, updated_at) VALUES \n" val insertOffersQuery = insertOffers + insertedOffersValues.joinToString(",\n") + "\n RETURNING id;" - val createdOfferIds: List = em.createNativeQuery(insertOffersQuery).resultList as List -// println("result: $createdOfferIds") + + @Suppress("UNCHECKED_CAST") + val createdOfferIds: List = entityManager + .createNativeQuery(insertOffersQuery) + .resultList as List + println("result: $createdOfferIds") // create tags val insertTags = "INSERT INTO offer_tags (offer_id, tags, tags_key) VALUES \n" - val insertedOfferTagsValues = offersForInserting.mapIndexed { index, offer -> + val insertedOfferTagsValues = offers.mapIndexed { index, offer -> offer.tags.map { "( ${createdOfferIds[index]}, '${it.value}', '${it.key}' )" } }.flatten().joinToString(",\n") val insertOfferTagsQuery = insertTags + insertedOfferTagsValues - em.createNativeQuery(insertOfferTagsQuery).executeUpdate() + entityManager.createNativeQuery(insertOfferTagsQuery).executeUpdate() // compare val insertCompare = "INSERT INTO offer_compare (offer_id, compare, compare_key) VALUES \n" - val insertedOfferCompareValues = offersForInserting.mapIndexed { index, offer -> + val insertedOfferCompareValues = offers.mapIndexed { index, offer -> offer.compare.map { "( ${createdOfferIds[index]}, '${it.value}', '${it.key}' )" } }.flatten().joinToString(",\n") val insertOfferCompareQuery = insertCompare + insertedOfferCompareValues - em.createNativeQuery(insertOfferCompareQuery).executeUpdate() + entityManager.createNativeQuery(insertOfferCompareQuery).executeUpdate() // rules val insertRules = "INSERT INTO offer_rules (offer_id, rules, rules_key) VALUES \n" - val insertedOfferRulesValues = offersForInserting.mapIndexed { index, offer -> + val insertedOfferRulesValues = offers.mapIndexed { index, offer -> offer.rules.map { "( ${createdOfferIds[index]}, ${it.value.ordinal}, '${it.key}' )" } }.flatten().joinToString(",\n") val insertOfferRulesQuery = insertRules + insertedOfferRulesValues - em.createNativeQuery(insertOfferRulesQuery).executeUpdate() + entityManager.createNativeQuery(insertOfferRulesQuery).executeUpdate() val ids = createdOfferIds.joinToString(", ") val query = "SELECT * FROM offer WHERE offer.id IN ($ids)" - result = em.createNativeQuery(query, Offer::class.java).resultList as List + @Suppress("UNCHECKED_CAST") + result = entityManager.createNativeQuery(query, Offer::class.java).resultList as List } println(" -- save all (native query) timing $springQueryTiming") } From 4988e08f8dda4cac17ca0e063fe06c69fab5271b Mon Sep 17 00:00:00 2001 From: batalin Date: Fri, 1 Nov 2019 14:16:50 +0200 Subject: [PATCH 03/14] attempt to delete for update --- .../offer/PostgresOfferRepositoryImpl.kt | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt b/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt index d6871cf5..05fa98cd 100644 --- a/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt +++ b/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt @@ -56,14 +56,24 @@ class PostgresOfferRepositoryImpl( // prepare for delete val offerForDeletion = offers.filter { it.id != 0L } if (offerForDeletion.isNotEmpty()) { - val ids = offerForDeletion.joinToString(", ") - var query = "DELETE FROM offer where offer.id IN ($ids)" - entityManager.createNativeQuery(query) .executeUpdate() - query = "DELETE FROM offer_tags where offer_tags.offer_id IN ($ids)" + val ids = offerForDeletion.map { it.id }.joinToString(", ") + + var query = "DELETE FROM offer_tags where offer_tags.offer_id IN ($ids)" + entityManager.createNativeQuery(query).executeUpdate() + query = "DELETE FROM offer_compare where offer_compare.offer_id IN ($ids)" entityManager.createNativeQuery(query).executeUpdate() - query = "DELETE FROM offer_compare where offer_compare.offer.id IN ($ids)" + query = "DELETE FROM offer_rules where offer_rules.offer_id IN ($ids)" entityManager.createNativeQuery(query).executeUpdate() - query = "DELETE FROM offer_rules where offer_rules.offer.id IN ($ids)" + + // delete price rules + query = "SELECT id FROM offer_price WHERE offer_price.offer_id IN ($ids)" + @Suppress("UNCHECKED_CAST") + var offerPriceIdsForDeletion = entityManager.createNativeQuery(query).resultList as List + query = "SELECT id" + + // delete price + + query = "DELETE FROM offer where offer.id IN ($ids)" entityManager.createNativeQuery(query).executeUpdate() } @@ -71,7 +81,7 @@ class PostgresOfferRepositoryImpl( val isoPattern = "yyyy-MM-dd'T'HH:mm:ss'Z'" val updatedAt = SimpleDateFormat(isoPattern).format(it.updatedAt) val createdAt = SimpleDateFormat(isoPattern).format(it.createdAt) - val id = if (it.id == 0L) "(nextval('offer_id_seq')," else it.id.toString() + val id = if (it.id == 0L) "(nextval('offer_id_seq')" else it.id.toString() "$id, '${it.title}', '${it.description}', '${it.owner}', " + "'${it.imageUrl}', '${it.worth}', '$createdAt', '$updatedAt')" From b4f2abd1d2ace89f044a1973aff731df7d000d01 Mon Sep 17 00:00:00 2001 From: batalin Date: Fri, 1 Nov 2019 16:04:28 +0200 Subject: [PATCH 04/14] new insert/update combined query --- .../offer/PostgresOfferRepositoryImpl.kt | 80 ++++++++----------- 1 file changed, 34 insertions(+), 46 deletions(-) diff --git a/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt b/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt index 05fa98cd..d8aa63c1 100644 --- a/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt +++ b/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt @@ -48,83 +48,71 @@ class PostgresOfferRepositoryImpl( } @Transactional override fun saveAll(offers: List): List { - // native spring implementation - // val result = repository.saveAll(offers).toList() - // return syncElementCollections(result) var result: List = listOf() - // prepare for delete - val offerForDeletion = offers.filter { it.id != 0L } - if (offerForDeletion.isNotEmpty()) { - val ids = offerForDeletion.map { it.id }.joinToString(", ") - - var query = "DELETE FROM offer_tags where offer_tags.offer_id IN ($ids)" - entityManager.createNativeQuery(query).executeUpdate() - query = "DELETE FROM offer_compare where offer_compare.offer_id IN ($ids)" - entityManager.createNativeQuery(query).executeUpdate() - query = "DELETE FROM offer_rules where offer_rules.offer_id IN ($ids)" - entityManager.createNativeQuery(query).executeUpdate() - - // delete price rules - query = "SELECT id FROM offer_price WHERE offer_price.offer_id IN ($ids)" - @Suppress("UNCHECKED_CAST") - var offerPriceIdsForDeletion = entityManager.createNativeQuery(query).resultList as List - query = "SELECT id" - - // delete price - - query = "DELETE FROM offer where offer.id IN ($ids)" - entityManager.createNativeQuery(query).executeUpdate() - } - - val insertedOffersValues = offers.map { + val values = offers.map { val isoPattern = "yyyy-MM-dd'T'HH:mm:ss'Z'" val updatedAt = SimpleDateFormat(isoPattern).format(it.updatedAt) val createdAt = SimpleDateFormat(isoPattern).format(it.createdAt) - val id = if (it.id == 0L) "(nextval('offer_id_seq')" else it.id.toString() + val id = if (it.id == 0L) "nextval('offer_id_seq')" else it.id.toString() - "$id, '${it.title}', '${it.description}', '${it.owner}', " + + "($id, '${it.title}', '${it.description}', '${it.owner}', " + "'${it.imageUrl}', '${it.worth}', '$createdAt', '$updatedAt')" } - if (insertedOffersValues.isNotEmpty()) { + if (values.isNotEmpty()) { val springQueryTiming = measureTimeMillis { // create offers val insertOffers = "INSERT INTO offer " + "(id, title, description, owner, image_url, worth, created_at, updated_at) VALUES \n" - val insertOffersQuery = insertOffers + insertedOffersValues.joinToString(",\n") + - "\n RETURNING id;" + val insertOffersQuery = insertOffers + values.joinToString(",\n") + + "\n ON CONFLICT (id) DO UPDATE SET\n" + + " title = EXCLUDED.title,\n" + + " description = EXCLUDED.description,\n" + + " owner = EXCLUDED.owner,\n" + + " image_url = EXCLUDED.image_url,\n" + + " worth = EXCLUDED.worth,\n" + + " updated_at = EXCLUDED.updated_at\n" + + "RETURNING id;" @Suppress("UNCHECKED_CAST") - val createdOfferIds: List = entityManager + val insertedOfferIds: List = entityManager .createNativeQuery(insertOffersQuery) .resultList as List - println("result: $createdOfferIds") + println("result: $insertedOfferIds") - // create tags val insertTags = "INSERT INTO offer_tags (offer_id, tags, tags_key) VALUES \n" val insertedOfferTagsValues = offers.mapIndexed { index, offer -> - offer.tags.map { "( ${createdOfferIds[index]}, '${it.value}', '${it.key}' )" } + offer.tags.map { "( ${insertedOfferIds[index]}, '${it.value}', '${it.key}' )" } }.flatten().joinToString(",\n") - val insertOfferTagsQuery = insertTags + insertedOfferTagsValues + val ifConflictPartTagsQuery = "\nON CONFLICT ON CONSTRAINT offer_tags_pkey DO UPDATE SET\n" + + " tags_key = EXCLUDED.tags_key,\n" + + " tags = EXCLUDED.tags" + val insertOfferTagsQuery = insertTags + insertedOfferTagsValues + ifConflictPartTagsQuery entityManager.createNativeQuery(insertOfferTagsQuery).executeUpdate() - // compare val insertCompare = "INSERT INTO offer_compare (offer_id, compare, compare_key) VALUES \n" val insertedOfferCompareValues = offers.mapIndexed { index, offer -> - offer.compare.map { "( ${createdOfferIds[index]}, '${it.value}', '${it.key}' )" } + offer.compare.map { "( ${insertedOfferIds[index]}, '${it.value}', '${it.key}' )" } }.flatten().joinToString(",\n") - val insertOfferCompareQuery = insertCompare + insertedOfferCompareValues - entityManager.createNativeQuery(insertOfferCompareQuery).executeUpdate() + val ifConflictOfferCompare = "\n ON CONFLICT ON CONSTRAINT offer_compare_pkey " + + "DO UPDATE SET\n" + + " compare = EXCLUDED.compare,\n" + + " compare_key = EXCLUDED.compare_key" + val offerCompareQuery = insertCompare + insertedOfferCompareValues + ifConflictOfferCompare + entityManager.createNativeQuery(offerCompareQuery).executeUpdate() - // rules val insertRules = "INSERT INTO offer_rules (offer_id, rules, rules_key) VALUES \n" val insertedOfferRulesValues = offers.mapIndexed { index, offer -> - offer.rules.map { "( ${createdOfferIds[index]}, ${it.value.ordinal}, '${it.key}' )" } + offer.rules.map { "( ${insertedOfferIds[index]}, ${it.value.ordinal}, '${it.key}' )" } }.flatten().joinToString(",\n") - val insertOfferRulesQuery = insertRules + insertedOfferRulesValues + val ifConflictOfferRules = "\n ON CONFLICT ON CONSTRAINT offer_rules_pkey " + + "DO UPDATE SET\n" + + " rules = EXCLUDED.rules,\n" + + " rules_key = EXCLUDED.rules_key" + val insertOfferRulesQuery = insertRules + insertedOfferRulesValues + ifConflictOfferRules entityManager.createNativeQuery(insertOfferRulesQuery).executeUpdate() - val ids = createdOfferIds.joinToString(", ") + val ids = insertedOfferIds.joinToString(", ") val query = "SELECT * FROM offer WHERE offer.id IN ($ids)" @Suppress("UNCHECKED_CAST") result = entityManager.createNativeQuery(query, Offer::class.java).resultList as List From 38d241ecefba4b3a97bbd4868d8eb8d81078de14 Mon Sep 17 00:00:00 2001 From: batalin Date: Fri, 1 Nov 2019 16:06:30 +0200 Subject: [PATCH 05/14] clean up --- .../offer/PostgresOfferRepositoryImpl.kt | 112 +++++++++--------- 1 file changed, 53 insertions(+), 59 deletions(-) diff --git a/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt b/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt index d8aa63c1..1e7dc00f 100644 --- a/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt +++ b/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt @@ -48,7 +48,6 @@ class PostgresOfferRepositoryImpl( } @Transactional override fun saveAll(offers: List): List { - var result: List = listOf() val values = offers.map { val isoPattern = "yyyy-MM-dd'T'HH:mm:ss'Z'" @@ -60,64 +59,59 @@ class PostgresOfferRepositoryImpl( "'${it.imageUrl}', '${it.worth}', '$createdAt', '$updatedAt')" } if (values.isNotEmpty()) { - val springQueryTiming = measureTimeMillis { - // create offers - val insertOffers = "INSERT INTO offer " + - "(id, title, description, owner, image_url, worth, created_at, updated_at) VALUES \n" - val insertOffersQuery = insertOffers + values.joinToString(",\n") + - "\n ON CONFLICT (id) DO UPDATE SET\n" + - " title = EXCLUDED.title,\n" + - " description = EXCLUDED.description,\n" + - " owner = EXCLUDED.owner,\n" + - " image_url = EXCLUDED.image_url,\n" + - " worth = EXCLUDED.worth,\n" + - " updated_at = EXCLUDED.updated_at\n" + - "RETURNING id;" - - @Suppress("UNCHECKED_CAST") - val insertedOfferIds: List = entityManager - .createNativeQuery(insertOffersQuery) - .resultList as List - println("result: $insertedOfferIds") - - val insertTags = "INSERT INTO offer_tags (offer_id, tags, tags_key) VALUES \n" - val insertedOfferTagsValues = offers.mapIndexed { index, offer -> - offer.tags.map { "( ${insertedOfferIds[index]}, '${it.value}', '${it.key}' )" } - }.flatten().joinToString(",\n") - val ifConflictPartTagsQuery = "\nON CONFLICT ON CONSTRAINT offer_tags_pkey DO UPDATE SET\n" + - " tags_key = EXCLUDED.tags_key,\n" + - " tags = EXCLUDED.tags" - val insertOfferTagsQuery = insertTags + insertedOfferTagsValues + ifConflictPartTagsQuery - entityManager.createNativeQuery(insertOfferTagsQuery).executeUpdate() - - val insertCompare = "INSERT INTO offer_compare (offer_id, compare, compare_key) VALUES \n" - val insertedOfferCompareValues = offers.mapIndexed { index, offer -> - offer.compare.map { "( ${insertedOfferIds[index]}, '${it.value}', '${it.key}' )" } - }.flatten().joinToString(",\n") - val ifConflictOfferCompare = "\n ON CONFLICT ON CONSTRAINT offer_compare_pkey " + - "DO UPDATE SET\n" + - " compare = EXCLUDED.compare,\n" + - " compare_key = EXCLUDED.compare_key" - val offerCompareQuery = insertCompare + insertedOfferCompareValues + ifConflictOfferCompare - entityManager.createNativeQuery(offerCompareQuery).executeUpdate() - - val insertRules = "INSERT INTO offer_rules (offer_id, rules, rules_key) VALUES \n" - val insertedOfferRulesValues = offers.mapIndexed { index, offer -> - offer.rules.map { "( ${insertedOfferIds[index]}, ${it.value.ordinal}, '${it.key}' )" } - }.flatten().joinToString(",\n") - val ifConflictOfferRules = "\n ON CONFLICT ON CONSTRAINT offer_rules_pkey " + - "DO UPDATE SET\n" + - " rules = EXCLUDED.rules,\n" + - " rules_key = EXCLUDED.rules_key" - val insertOfferRulesQuery = insertRules + insertedOfferRulesValues + ifConflictOfferRules - entityManager.createNativeQuery(insertOfferRulesQuery).executeUpdate() - - val ids = insertedOfferIds.joinToString(", ") - val query = "SELECT * FROM offer WHERE offer.id IN ($ids)" - @Suppress("UNCHECKED_CAST") - result = entityManager.createNativeQuery(query, Offer::class.java).resultList as List - } - println(" -- save all (native query) timing $springQueryTiming") + val insertOffers = "INSERT INTO offer " + + "(id, title, description, owner, image_url, worth, created_at, updated_at) VALUES \n" + val insertOffersQuery = insertOffers + values.joinToString(",\n") + + "\n ON CONFLICT (id) DO UPDATE SET\n" + + " title = EXCLUDED.title,\n" + + " description = EXCLUDED.description,\n" + + " owner = EXCLUDED.owner,\n" + + " image_url = EXCLUDED.image_url,\n" + + " worth = EXCLUDED.worth,\n" + + " updated_at = EXCLUDED.updated_at\n" + + "RETURNING id;" + + @Suppress("UNCHECKED_CAST") + val insertedOfferIds: List = entityManager + .createNativeQuery(insertOffersQuery) + .resultList as List + + val insertTags = "INSERT INTO offer_tags (offer_id, tags, tags_key) VALUES \n" + val insertedOfferTagsValues = offers.mapIndexed { index, offer -> + offer.tags.map { "( ${insertedOfferIds[index]}, '${it.value}', '${it.key}' )" } + }.flatten().joinToString(",\n") + val ifConflictPartTagsQuery = "\nON CONFLICT ON CONSTRAINT offer_tags_pkey DO UPDATE SET\n" + + " tags_key = EXCLUDED.tags_key,\n" + + " tags = EXCLUDED.tags" + val insertOfferTagsQuery = insertTags + insertedOfferTagsValues + ifConflictPartTagsQuery + entityManager.createNativeQuery(insertOfferTagsQuery).executeUpdate() + + val insertCompare = "INSERT INTO offer_compare (offer_id, compare, compare_key) VALUES \n" + val insertedOfferCompareValues = offers.mapIndexed { index, offer -> + offer.compare.map { "( ${insertedOfferIds[index]}, '${it.value}', '${it.key}' )" } + }.flatten().joinToString(",\n") + val ifConflictOfferCompare = "\n ON CONFLICT ON CONSTRAINT offer_compare_pkey " + + "DO UPDATE SET\n" + + " compare = EXCLUDED.compare,\n" + + " compare_key = EXCLUDED.compare_key" + val offerCompareQuery = insertCompare + insertedOfferCompareValues + ifConflictOfferCompare + entityManager.createNativeQuery(offerCompareQuery).executeUpdate() + + val insertRules = "INSERT INTO offer_rules (offer_id, rules, rules_key) VALUES \n" + val insertedOfferRulesValues = offers.mapIndexed { index, offer -> + offer.rules.map { "( ${insertedOfferIds[index]}, ${it.value.ordinal}, '${it.key}' )" } + }.flatten().joinToString(",\n") + val ifConflictOfferRules = "\n ON CONFLICT ON CONSTRAINT offer_rules_pkey " + + "DO UPDATE SET\n" + + " rules = EXCLUDED.rules,\n" + + " rules_key = EXCLUDED.rules_key" + val insertOfferRulesQuery = insertRules + insertedOfferRulesValues + ifConflictOfferRules + entityManager.createNativeQuery(insertOfferRulesQuery).executeUpdate() + + val ids = insertedOfferIds.joinToString(", ") + val query = "SELECT * FROM offer WHERE offer.id IN ($ids)" + @Suppress("UNCHECKED_CAST") + result = entityManager.createNativeQuery(query, Offer::class.java).resultList as List } return syncElementCollections(result) } From aebafcd4b6b865d9dd1fc87dae536fe174a4d6c2 Mon Sep 17 00:00:00 2001 From: batalin Date: Tue, 5 Nov 2019 18:24:05 +0200 Subject: [PATCH 06/14] empty dependencies check and fixed sequence of returning ids --- .../offer/PostgresOfferRepositoryImpl.kt | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt b/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt index 1e7dc00f..ed76994a 100644 --- a/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt +++ b/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt @@ -80,38 +80,46 @@ class PostgresOfferRepositoryImpl( val insertedOfferTagsValues = offers.mapIndexed { index, offer -> offer.tags.map { "( ${insertedOfferIds[index]}, '${it.value}', '${it.key}' )" } }.flatten().joinToString(",\n") - val ifConflictPartTagsQuery = "\nON CONFLICT ON CONSTRAINT offer_tags_pkey DO UPDATE SET\n" + - " tags_key = EXCLUDED.tags_key,\n" + - " tags = EXCLUDED.tags" - val insertOfferTagsQuery = insertTags + insertedOfferTagsValues + ifConflictPartTagsQuery - entityManager.createNativeQuery(insertOfferTagsQuery).executeUpdate() + if (insertedOfferTagsValues.isNotEmpty()) { + val ifConflictPartTagsQuery = "\nON CONFLICT ON CONSTRAINT offer_tags_pkey DO UPDATE SET\n" + + " tags_key = EXCLUDED.tags_key,\n" + + " tags = EXCLUDED.tags" + val insertOfferTagsQuery = insertTags + insertedOfferTagsValues + ifConflictPartTagsQuery + entityManager.createNativeQuery(insertOfferTagsQuery).executeUpdate() + } val insertCompare = "INSERT INTO offer_compare (offer_id, compare, compare_key) VALUES \n" val insertedOfferCompareValues = offers.mapIndexed { index, offer -> offer.compare.map { "( ${insertedOfferIds[index]}, '${it.value}', '${it.key}' )" } }.flatten().joinToString(",\n") - val ifConflictOfferCompare = "\n ON CONFLICT ON CONSTRAINT offer_compare_pkey " + - "DO UPDATE SET\n" + - " compare = EXCLUDED.compare,\n" + - " compare_key = EXCLUDED.compare_key" - val offerCompareQuery = insertCompare + insertedOfferCompareValues + ifConflictOfferCompare - entityManager.createNativeQuery(offerCompareQuery).executeUpdate() + if (insertedOfferCompareValues.isNotEmpty()) { + val ifConflictOfferCompare = "\n ON CONFLICT ON CONSTRAINT offer_compare_pkey " + + "DO UPDATE SET\n" + + " compare = EXCLUDED.compare,\n" + + " compare_key = EXCLUDED.compare_key" + val offerCompareQuery = insertCompare + insertedOfferCompareValues + ifConflictOfferCompare + entityManager.createNativeQuery(offerCompareQuery).executeUpdate() + } val insertRules = "INSERT INTO offer_rules (offer_id, rules, rules_key) VALUES \n" val insertedOfferRulesValues = offers.mapIndexed { index, offer -> offer.rules.map { "( ${insertedOfferIds[index]}, ${it.value.ordinal}, '${it.key}' )" } }.flatten().joinToString(",\n") - val ifConflictOfferRules = "\n ON CONFLICT ON CONSTRAINT offer_rules_pkey " + - "DO UPDATE SET\n" + - " rules = EXCLUDED.rules,\n" + - " rules_key = EXCLUDED.rules_key" - val insertOfferRulesQuery = insertRules + insertedOfferRulesValues + ifConflictOfferRules - entityManager.createNativeQuery(insertOfferRulesQuery).executeUpdate() + if (insertedOfferRulesValues.isNotEmpty()) { + val ifConflictOfferRules = "\n ON CONFLICT ON CONSTRAINT offer_rules_pkey " + + "DO UPDATE SET\n" + + " rules = EXCLUDED.rules,\n" + + " rules_key = EXCLUDED.rules_key" + val insertOfferRulesQuery = insertRules + insertedOfferRulesValues + ifConflictOfferRules + entityManager.createNativeQuery(insertOfferRulesQuery).executeUpdate() + } val ids = insertedOfferIds.joinToString(", ") val query = "SELECT * FROM offer WHERE offer.id IN ($ids)" @Suppress("UNCHECKED_CAST") - result = entityManager.createNativeQuery(query, Offer::class.java).resultList as List + val wrongSortedResult = entityManager.createNativeQuery(query, Offer::class.java).resultList as List + val wrongSortedResultAsMap = wrongSortedResult.map { it.id to it}.toMap() + result = insertedOfferIds.map { wrongSortedResultAsMap[it] ?: error("was not find bt ids") } } return syncElementCollections(result) } From 8998afa3203c0f1fda09a9917f2b1b27fe0f82e1 Mon Sep 17 00:00:00 2001 From: batalin Date: Thu, 7 Nov 2019 15:32:21 +0200 Subject: [PATCH 07/14] cleanup before update for all entities --- .../offer/PostgresOfferRepositoryImpl.kt | 17 ++++++++++++++ .../repository/price/OfferPriceRepository.kt | 2 +- .../price/PostgresOfferPriceRepositoryImpl.kt | 22 +++++++++++++++++-- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt b/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt index ed76994a..479ebafc 100644 --- a/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt +++ b/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt @@ -76,6 +76,12 @@ class PostgresOfferRepositoryImpl( .createNativeQuery(insertOffersQuery) .resultList as List + // cleanup tags for the all offers in the request + val offerInByIds = insertedOfferIds.joinToString(", ") + val cleanUpTagsQuery = "DELETE FROM offer_tags\n" + + "WHERE offer_tags.offer_id IN ($offerInByIds);" + entityManager.createNativeQuery(cleanUpTagsQuery).executeUpdate() + val insertTags = "INSERT INTO offer_tags (offer_id, tags, tags_key) VALUES \n" val insertedOfferTagsValues = offers.mapIndexed { index, offer -> offer.tags.map { "( ${insertedOfferIds[index]}, '${it.value}', '${it.key}' )" } @@ -88,6 +94,11 @@ class PostgresOfferRepositoryImpl( entityManager.createNativeQuery(insertOfferTagsQuery).executeUpdate() } + // cleanup compare for the all offers in the request + val cleanUpCompareQuery = "DELETE FROM offer_compare\n" + + "WHERE offer_compare.offer_id IN ($offerInByIds);" + entityManager.createNativeQuery(cleanUpCompareQuery).executeUpdate() + val insertCompare = "INSERT INTO offer_compare (offer_id, compare, compare_key) VALUES \n" val insertedOfferCompareValues = offers.mapIndexed { index, offer -> offer.compare.map { "( ${insertedOfferIds[index]}, '${it.value}', '${it.key}' )" } @@ -101,6 +112,11 @@ class PostgresOfferRepositoryImpl( entityManager.createNativeQuery(offerCompareQuery).executeUpdate() } + // cleanup rules for the all offers in the request + val cleanUpRulesQuery = "DELETE FROM offer_rules\n" + + "WHERE offer_rules.offer_id IN ($offerInByIds);" + entityManager.createNativeQuery(cleanUpRulesQuery).executeUpdate() + val insertRules = "INSERT INTO offer_rules (offer_id, rules, rules_key) VALUES \n" val insertedOfferRulesValues = offers.mapIndexed { index, offer -> offer.rules.map { "( ${insertedOfferIds[index]}, ${it.value.ordinal}, '${it.key}' )" } @@ -114,6 +130,7 @@ class PostgresOfferRepositoryImpl( entityManager.createNativeQuery(insertOfferRulesQuery).executeUpdate() } + // for proper return data purpose only val ids = insertedOfferIds.joinToString(", ") val query = "SELECT * FROM offer WHERE offer.id IN ($ids)" @Suppress("UNCHECKED_CAST") diff --git a/src/main/kotlin/com/bitclave/node/repository/price/OfferPriceRepository.kt b/src/main/kotlin/com/bitclave/node/repository/price/OfferPriceRepository.kt index 8f5a6444..29d0c3bd 100644 --- a/src/main/kotlin/com/bitclave/node/repository/price/OfferPriceRepository.kt +++ b/src/main/kotlin/com/bitclave/node/repository/price/OfferPriceRepository.kt @@ -5,5 +5,5 @@ import com.bitclave.node.repository.entities.OfferPrice interface OfferPriceRepository { fun savePrices(offer: Offer, prices: List): List - fun saveAllPrices(prices: List): List + fun saveAllPrices(prices: List, offerIds: List): List } diff --git a/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt b/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt index 62a9e20e..80385324 100644 --- a/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt +++ b/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt @@ -7,14 +7,32 @@ import com.bitclave.node.repository.priceRule.OfferPriceRulesCrudRepository import com.bitclave.node.services.errors.DataNotSavedException import org.springframework.beans.factory.annotation.Qualifier import org.springframework.stereotype.Component +import org.springframework.transaction.annotation.Transactional +import javax.persistence.EntityManager @Component @Qualifier("postgres") class PostgresOfferPriceRepositoryImpl( val repository: OfferPriceCrudRepository, - val rulesRepository: OfferPriceRulesCrudRepository + val rulesRepository: OfferPriceRulesCrudRepository, + val entityManager: EntityManager ) : OfferPriceRepository { - override fun saveAllPrices(prices: List): List { + + @Transactional + override fun saveAllPrices(prices: List, offerIds: List): List { + + val offerIdsIn = offerIds.joinToString(", ") + val priceIds = prices.map { it.id }.joinToString(", ") + + val cleanUpQuery = "DELETE FROM offer_price_rules r\n" + + "WHERE r.offer_price_id IN (\n" + + " select id from offer_price p\n" + + " where p.offer_id in ($offerIdsIn) AND p.id NOT IN ($priceIds)\n" + + ");" + + "DELETE FROM offer_price p WHERE p.offer_id in ($offerIdsIn) AND p.id NOT IN ($priceIds)" + + entityManager.createNativeQuery(cleanUpQuery).executeUpdate() + val savedPrices = repository.saveAll(prices) val rules = savedPrices.mapIndexed { index, offerPrice -> From 408a25cef2752a9682e2250cf26c967e020eec9f Mon Sep 17 00:00:00 2001 From: batalin Date: Thu, 7 Nov 2019 17:33:15 +0200 Subject: [PATCH 08/14] with cleanup code --- .../price/PostgresOfferPriceRepositoryImpl.kt | 25 +++++++++++++------ .../bitclave/node/services/v1/OfferService.kt | 5 ++-- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt b/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt index 80385324..f495b622 100644 --- a/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt +++ b/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt @@ -8,6 +8,7 @@ import com.bitclave.node.services.errors.DataNotSavedException import org.springframework.beans.factory.annotation.Qualifier import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional +import java.lang.RuntimeException import javax.persistence.EntityManager @Component @@ -22,16 +23,24 @@ class PostgresOfferPriceRepositoryImpl( override fun saveAllPrices(prices: List, offerIds: List): List { val offerIdsIn = offerIds.joinToString(", ") - val priceIds = prices.map { it.id }.joinToString(", ") +// val priceIds = prices.map { it.id }.joinToString(", ") - val cleanUpQuery = "DELETE FROM offer_price_rules r\n" + - "WHERE r.offer_price_id IN (\n" + - " select id from offer_price p\n" + - " where p.offer_id in ($offerIdsIn) AND p.id NOT IN ($priceIds)\n" + - ");" + - "DELETE FROM offer_price p WHERE p.offer_id in ($offerIdsIn) AND p.id NOT IN ($priceIds)" +// val cleanUpQuery = "DELETE FROM offer_price_rules r\n" + +// "WHERE r.offer_price_id IN (\n" + +// " select id from offer_price p\n" + +// " where p.offer_id in ($offerIdsIn) AND p.id NOT IN ($priceIds)\n" + +// ");" + +// "DELETE FROM offer_price p WHERE p.offer_id in ($offerIdsIn) AND p.id NOT IN ($priceIds)" +// +// entityManager.createNativeQuery(cleanUpQuery).executeUpdate() + val checkQuery = "SELECT id FROM offer_price p WHERE p.offer_id in ($offerIdsIn)" + @Suppress("UNCHECKED_CAST") + val formerIds = entityManager.createNativeQuery(checkQuery).resultList as List + + if (formerIds.size > prices.size) { + throw RuntimeException("attempt to reduce prices in bulk of offer") + } - entityManager.createNativeQuery(cleanUpQuery).executeUpdate() val savedPrices = repository.saveAll(prices) diff --git a/src/main/kotlin/com/bitclave/node/services/v1/OfferService.kt b/src/main/kotlin/com/bitclave/node/services/v1/OfferService.kt index c4a79be1..b4400569 100644 --- a/src/main/kotlin/com/bitclave/node/services/v1/OfferService.kt +++ b/src/main/kotlin/com/bitclave/node/services/v1/OfferService.kt @@ -152,7 +152,8 @@ class OfferService( }.flatten() val saveAllPricesTiming = measureTimeMillis { - offerPriceRepository.changeStrategy(strategy).saveAllPrices(prices) + val ids = result.map { it.id } + offerPriceRepository.changeStrategy(strategy).saveAllPrices(prices, ids) } Logger.debug(" - save all prices timing $saveAllPricesTiming") @@ -162,7 +163,7 @@ class OfferService( offerSearchService.deleteByOfferIds(offersIdsForCleanupOfferSearches, strategy) } } - Logger.debug(" - delete all offer searches by offerId timing $deleteOfferSearchTiming") +// Logger.debug(" - delete all offer searches by offerId timing $deleteOfferSearchTiming") result.map { it.id } }) From bd8310a1f12496913f45bb4171a2cf96ecdb593e Mon Sep 17 00:00:00 2001 From: batalin Date: Thu, 7 Nov 2019 17:34:09 +0200 Subject: [PATCH 09/14] cleanup code --- .../price/PostgresOfferPriceRepositoryImpl.kt | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt b/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt index f495b622..adebc01d 100644 --- a/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt +++ b/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt @@ -23,17 +23,8 @@ class PostgresOfferPriceRepositoryImpl( override fun saveAllPrices(prices: List, offerIds: List): List { val offerIdsIn = offerIds.joinToString(", ") -// val priceIds = prices.map { it.id }.joinToString(", ") - -// val cleanUpQuery = "DELETE FROM offer_price_rules r\n" + -// "WHERE r.offer_price_id IN (\n" + -// " select id from offer_price p\n" + -// " where p.offer_id in ($offerIdsIn) AND p.id NOT IN ($priceIds)\n" + -// ");" + -// "DELETE FROM offer_price p WHERE p.offer_id in ($offerIdsIn) AND p.id NOT IN ($priceIds)" -// -// entityManager.createNativeQuery(cleanUpQuery).executeUpdate() val checkQuery = "SELECT id FROM offer_price p WHERE p.offer_id in ($offerIdsIn)" + @Suppress("UNCHECKED_CAST") val formerIds = entityManager.createNativeQuery(checkQuery).resultList as List @@ -41,7 +32,6 @@ class PostgresOfferPriceRepositoryImpl( throw RuntimeException("attempt to reduce prices in bulk of offer") } - val savedPrices = repository.saveAll(prices) val rules = savedPrices.mapIndexed { index, offerPrice -> From 342ec178f726e85d4bccd6485a43646153ba66e4 Mon Sep 17 00:00:00 2001 From: batalin Date: Thu, 7 Nov 2019 20:43:19 +0200 Subject: [PATCH 10/14] bad characters in SQL query fix --- .../offer/PostgresOfferRepositoryImpl.kt | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt b/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt index 479ebafc..46c3c439 100644 --- a/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt +++ b/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt @@ -55,7 +55,10 @@ class PostgresOfferRepositoryImpl( val createdAt = SimpleDateFormat(isoPattern).format(it.createdAt) val id = if (it.id == 0L) "nextval('offer_id_seq')" else it.id.toString() - "($id, '${it.title}', '${it.description}', '${it.owner}', " + + "($id, " + + "'${it.title.replace("'", "''")}'," + + " '${it.description.replace("'", "''")}'," + + " '${it.owner.replace("'", "''")}', " + "'${it.imageUrl}', '${it.worth}', '$createdAt', '$updatedAt')" } if (values.isNotEmpty()) { @@ -84,7 +87,9 @@ class PostgresOfferRepositoryImpl( val insertTags = "INSERT INTO offer_tags (offer_id, tags, tags_key) VALUES \n" val insertedOfferTagsValues = offers.mapIndexed { index, offer -> - offer.tags.map { "( ${insertedOfferIds[index]}, '${it.value}', '${it.key}' )" } + offer.tags.map { "( ${insertedOfferIds[index]}, " + + "'${it.value.replace("'", "''")}', " + + "'${it.key.replace("'", "''")}' )" } }.flatten().joinToString(",\n") if (insertedOfferTagsValues.isNotEmpty()) { val ifConflictPartTagsQuery = "\nON CONFLICT ON CONSTRAINT offer_tags_pkey DO UPDATE SET\n" + @@ -101,7 +106,9 @@ class PostgresOfferRepositoryImpl( val insertCompare = "INSERT INTO offer_compare (offer_id, compare, compare_key) VALUES \n" val insertedOfferCompareValues = offers.mapIndexed { index, offer -> - offer.compare.map { "( ${insertedOfferIds[index]}, '${it.value}', '${it.key}' )" } + offer.compare.map { "( ${insertedOfferIds[index]}, " + + "'${it.value.replace("'", "''")}', " + + "'${it.key.replace("'", "''")}' )" } }.flatten().joinToString(",\n") if (insertedOfferCompareValues.isNotEmpty()) { val ifConflictOfferCompare = "\n ON CONFLICT ON CONSTRAINT offer_compare_pkey " + @@ -119,7 +126,8 @@ class PostgresOfferRepositoryImpl( val insertRules = "INSERT INTO offer_rules (offer_id, rules, rules_key) VALUES \n" val insertedOfferRulesValues = offers.mapIndexed { index, offer -> - offer.rules.map { "( ${insertedOfferIds[index]}, ${it.value.ordinal}, '${it.key}' )" } + offer.rules.map { "( ${insertedOfferIds[index]}, ${it.value.ordinal}, " + + "'${it.key.replace("'", "''")}' )" } }.flatten().joinToString(",\n") if (insertedOfferRulesValues.isNotEmpty()) { val ifConflictOfferRules = "\n ON CONFLICT ON CONSTRAINT offer_rules_pkey " + From 31534c1cf62b251cc1729da792772d7a4725ae10 Mon Sep 17 00:00:00 2001 From: batalin Date: Thu, 7 Nov 2019 20:43:58 +0200 Subject: [PATCH 11/14] comment logs --- src/main/kotlin/com/bitclave/node/services/v1/OfferService.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/com/bitclave/node/services/v1/OfferService.kt b/src/main/kotlin/com/bitclave/node/services/v1/OfferService.kt index b4400569..73bf1e6a 100644 --- a/src/main/kotlin/com/bitclave/node/services/v1/OfferService.kt +++ b/src/main/kotlin/com/bitclave/node/services/v1/OfferService.kt @@ -141,7 +141,7 @@ class OfferService( val saveAllTiming = measureTimeMillis { result = offerRepository.changeStrategy(strategy).saveAll(readyForSaveOffers) } - Logger.debug(" - save all timing $saveAllTiming") +// Logger.debug(" - save all timing $saveAllTiming") val prices = result.mapIndexed { index, offer -> readyForSaveOffers[index].offerPrices.map { offerPrice -> @@ -155,7 +155,7 @@ class OfferService( val ids = result.map { it.id } offerPriceRepository.changeStrategy(strategy).saveAllPrices(prices, ids) } - Logger.debug(" - save all prices timing $saveAllPricesTiming") +// Logger.debug(" - save all prices timing $saveAllPricesTiming") val offersIdsForCleanupOfferSearches = offers.filter { it.id != 0L }.map { it.id } val deleteOfferSearchTiming = measureTimeMillis { From a0a600245578fb6a60ebd78dc5d886b949feb3c1 Mon Sep 17 00:00:00 2001 From: batalin Date: Thu, 7 Nov 2019 20:44:50 +0200 Subject: [PATCH 12/14] get rid of reduce price case checker --- .../price/PostgresOfferPriceRepositoryImpl.kt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt b/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt index adebc01d..fb90c8e8 100644 --- a/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt +++ b/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt @@ -22,15 +22,15 @@ class PostgresOfferPriceRepositoryImpl( @Transactional override fun saveAllPrices(prices: List, offerIds: List): List { - val offerIdsIn = offerIds.joinToString(", ") - val checkQuery = "SELECT id FROM offer_price p WHERE p.offer_id in ($offerIdsIn)" +// val offerIdsIn = offerIds.joinToString(", ") +// val checkQuery = "SELECT id FROM offer_price p WHERE p.offer_id in ($offerIdsIn)" - @Suppress("UNCHECKED_CAST") - val formerIds = entityManager.createNativeQuery(checkQuery).resultList as List - - if (formerIds.size > prices.size) { - throw RuntimeException("attempt to reduce prices in bulk of offer") - } +// @Suppress("UNCHECKED_CAST") +// val formerIds = entityManager.createNativeQuery(checkQuery).resultList as List +// +// if (formerIds.size > prices.size) { +// throw RuntimeException("attempt to reduce prices in bulk of offer") +// } val savedPrices = repository.saveAll(prices) From db86423f52770dc6787676ebd3993acbaa67262a Mon Sep 17 00:00:00 2001 From: batalin Date: Thu, 7 Nov 2019 20:49:32 +0200 Subject: [PATCH 13/14] cleanup code --- .../repository/offer/PostgresOfferRepositoryImpl.kt | 2 +- .../price/PostgresOfferPriceRepositoryImpl.kt | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt b/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt index 46c3c439..321fd94e 100644 --- a/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt +++ b/src/main/kotlin/com/bitclave/node/repository/offer/PostgresOfferRepositoryImpl.kt @@ -143,7 +143,7 @@ class PostgresOfferRepositoryImpl( val query = "SELECT * FROM offer WHERE offer.id IN ($ids)" @Suppress("UNCHECKED_CAST") val wrongSortedResult = entityManager.createNativeQuery(query, Offer::class.java).resultList as List - val wrongSortedResultAsMap = wrongSortedResult.map { it.id to it}.toMap() + val wrongSortedResultAsMap = wrongSortedResult.map { it.id to it }.toMap() result = insertedOfferIds.map { wrongSortedResultAsMap[it] ?: error("was not find bt ids") } } return syncElementCollections(result) diff --git a/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt b/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt index fb90c8e8..d7e846d3 100644 --- a/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt +++ b/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt @@ -8,7 +8,6 @@ import com.bitclave.node.services.errors.DataNotSavedException import org.springframework.beans.factory.annotation.Qualifier import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional -import java.lang.RuntimeException import javax.persistence.EntityManager @Component @@ -22,16 +21,6 @@ class PostgresOfferPriceRepositoryImpl( @Transactional override fun saveAllPrices(prices: List, offerIds: List): List { -// val offerIdsIn = offerIds.joinToString(", ") -// val checkQuery = "SELECT id FROM offer_price p WHERE p.offer_id in ($offerIdsIn)" - -// @Suppress("UNCHECKED_CAST") -// val formerIds = entityManager.createNativeQuery(checkQuery).resultList as List -// -// if (formerIds.size > prices.size) { -// throw RuntimeException("attempt to reduce prices in bulk of offer") -// } - val savedPrices = repository.saveAll(prices) val rules = savedPrices.mapIndexed { index, offerPrice -> From db7fc27ed1aaa4da31ec74f2ee414b82adf59ab0 Mon Sep 17 00:00:00 2001 From: batalin Date: Thu, 7 Nov 2019 20:58:46 +0200 Subject: [PATCH 14/14] cleanup (restore save all prices) --- .../bitclave/node/repository/price/OfferPriceRepository.kt | 2 +- .../repository/price/PostgresOfferPriceRepositoryImpl.kt | 6 ++---- .../kotlin/com/bitclave/node/services/v1/OfferService.kt | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/com/bitclave/node/repository/price/OfferPriceRepository.kt b/src/main/kotlin/com/bitclave/node/repository/price/OfferPriceRepository.kt index 29d0c3bd..8f5a6444 100644 --- a/src/main/kotlin/com/bitclave/node/repository/price/OfferPriceRepository.kt +++ b/src/main/kotlin/com/bitclave/node/repository/price/OfferPriceRepository.kt @@ -5,5 +5,5 @@ import com.bitclave.node.repository.entities.OfferPrice interface OfferPriceRepository { fun savePrices(offer: Offer, prices: List): List - fun saveAllPrices(prices: List, offerIds: List): List + fun saveAllPrices(prices: List): List } diff --git a/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt b/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt index d7e846d3..d86265a6 100644 --- a/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt +++ b/src/main/kotlin/com/bitclave/node/repository/price/PostgresOfferPriceRepositoryImpl.kt @@ -8,18 +8,16 @@ import com.bitclave.node.services.errors.DataNotSavedException import org.springframework.beans.factory.annotation.Qualifier import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional -import javax.persistence.EntityManager @Component @Qualifier("postgres") class PostgresOfferPriceRepositoryImpl( val repository: OfferPriceCrudRepository, - val rulesRepository: OfferPriceRulesCrudRepository, - val entityManager: EntityManager + val rulesRepository: OfferPriceRulesCrudRepository ) : OfferPriceRepository { @Transactional - override fun saveAllPrices(prices: List, offerIds: List): List { + override fun saveAllPrices(prices: List): List { val savedPrices = repository.saveAll(prices) diff --git a/src/main/kotlin/com/bitclave/node/services/v1/OfferService.kt b/src/main/kotlin/com/bitclave/node/services/v1/OfferService.kt index 73bf1e6a..56837841 100644 --- a/src/main/kotlin/com/bitclave/node/services/v1/OfferService.kt +++ b/src/main/kotlin/com/bitclave/node/services/v1/OfferService.kt @@ -153,7 +153,7 @@ class OfferService( val saveAllPricesTiming = measureTimeMillis { val ids = result.map { it.id } - offerPriceRepository.changeStrategy(strategy).saveAllPrices(prices, ids) + offerPriceRepository.changeStrategy(strategy).saveAllPrices(prices) } // Logger.debug(" - save all prices timing $saveAllPricesTiming")