diff --git a/apps/labrinth/.sqlx/query-1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286.json b/apps/labrinth/.sqlx/query-1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286.json deleted file mode 100644 index 921f7f92d9..0000000000 --- a/apps/labrinth/.sqlx/query-1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n id,\n status AS \"status: PayoutStatus\"\n FROM payouts\n ORDER BY id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "status: PayoutStatus", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false - ] - }, - "hash": "1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286" -} diff --git a/apps/labrinth/.sqlx/query-b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5.json b/apps/labrinth/.sqlx/query-2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa.json similarity index 88% rename from apps/labrinth/.sqlx/query-b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5.json rename to apps/labrinth/.sqlx/query-2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa.json index 6142e7dcaf..32dcb10585 100644 --- a/apps/labrinth/.sqlx/query-b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5.json +++ b/apps/labrinth/.sqlx/query-2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,\n m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color\n FROM mods m\n WHERE m.status = ANY($1)\n GROUP BY m.id;\n ", + "query": "\n SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,\n m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color\n FROM mods m\n WHERE m.status = ANY($1) AND m.id > $2\n GROUP BY m.id\n LIMIT $3\n ", "describe": { "columns": [ { @@ -66,7 +66,9 @@ ], "parameters": { "Left": [ - "TextArray" + "TextArray", + "Int8", + "Int8" ] }, "nullable": [ @@ -84,5 +86,5 @@ true ] }, - "hash": "b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5" + "hash": "2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa" } diff --git a/apps/labrinth/.sqlx/query-b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3.json b/apps/labrinth/.sqlx/query-b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3.json deleted file mode 100644 index 89bd8147dc..0000000000 --- a/apps/labrinth/.sqlx/query-b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT status AS \"status: PayoutStatus\" FROM payouts WHERE id = 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "status: PayoutStatus", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false - ] - }, - "hash": "b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3" -} diff --git a/apps/labrinth/.sqlx/query-cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02.json b/apps/labrinth/.sqlx/query-cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02.json deleted file mode 100644 index 469c30168a..0000000000 --- a/apps/labrinth/.sqlx/query-cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO payouts (id, method, platform_id, status, user_id, amount, created)\n VALUES ($1, $2, $3, $4, $5, 10.0, NOW())\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Text", - "Text", - "Varchar", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02" -} diff --git a/apps/labrinth/.sqlx/query-cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606.json b/apps/labrinth/.sqlx/query-cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606.json deleted file mode 100644 index 52e020ebf2..0000000000 --- a/apps/labrinth/.sqlx/query-cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO payouts (id, method, platform_id, status, user_id, amount, created)\n VALUES ($1, $2, NULL, $3, $4, 10.00, NOW())\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Text", - "Varchar", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606" -} diff --git a/apps/labrinth/src/search/indexing/local_import.rs b/apps/labrinth/src/search/indexing/local_import.rs index e1620d8ce1..001984b29d 100644 --- a/apps/labrinth/src/search/indexing/local_import.rs +++ b/apps/labrinth/src/search/indexing/local_import.rs @@ -20,10 +20,12 @@ use crate::routes::v2_reroute; use crate::search::UploadSearchProject; use sqlx::postgres::PgPool; -pub async fn index_local( +pub async fn index_local_chunk( pool: &PgPool, -) -> Result, IndexingError> { - info!("Indexing local projects!"); + cursor: i64, + limit: i64, +) -> Result<(Vec, i64), IndexingError> { + info!(cursor = cursor, limit = limit, "Indexing local projects"); // todo: loaders, project type, game versions struct PartialProject { @@ -45,13 +47,16 @@ pub async fn index_local( SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows, m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color FROM mods m - WHERE m.status = ANY($1) - GROUP BY m.id; + WHERE m.status = ANY($1) AND m.id > $2 + GROUP BY m.id + LIMIT $3 ", &*crate::models::projects::ProjectStatus::iterator() .filter(|x| x.is_searchable()) .map(|x| x.to_string()) .collect::>(), + cursor, + limit, ) .fetch(pool) .map_ok(|m| { @@ -74,6 +79,10 @@ pub async fn index_local( let project_ids = db_projects.iter().map(|x| x.id.0).collect::>(); + let Some(largest) = project_ids.iter().max() else { + return Ok((vec![], i64::MAX)); + }; + struct PartialGallery { url: String, featured: bool, @@ -412,7 +421,7 @@ pub async fn index_local( } } - Ok(uploads) + Ok((uploads, *largest)) } struct PartialVersion { diff --git a/apps/labrinth/src/search/indexing/mod.rs b/apps/labrinth/src/search/indexing/mod.rs index 52c8c5d909..f9f3b3d161 100644 --- a/apps/labrinth/src/search/indexing/mod.rs +++ b/apps/labrinth/src/search/indexing/mod.rs @@ -8,7 +8,7 @@ use crate::search::{SearchConfig, UploadSearchProject}; use ariadne::ids::base62_impl::to_base62; use futures::StreamExt; use futures::stream::FuturesOrdered; -use local_import::index_local; +use local_import::index_local_chunk; use meilisearch_sdk::client::{Client, SwapIndexes}; use meilisearch_sdk::indexes::Index; use meilisearch_sdk::settings::{PaginationSetting, Settings}; @@ -34,8 +34,8 @@ pub enum IndexingError { // The chunk size for adding projects to the indexing database. If the request size // is too large (>10MiB) then the request fails with an error. This chunk size -// assumes a max average size of 4KiB per project to avoid this cap. -const MEILISEARCH_CHUNK_SIZE: usize = 10000000; +// assumes a max average size of 8KiB per project to avoid this cap. +const MEILISEARCH_CHUNK_SIZE: usize = 5000000; const TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); pub async fn remove_documents( @@ -115,15 +115,35 @@ pub async fn index_projects( .map(|x| x.field) .collect::>(); - let uploads = index_local(&pool).await?; + let mut cursor = 0; + let mut idx = 0; + let mut total = 0; - add_projects_batch_client( - &indices, - uploads, - all_loader_fields.clone(), - config, - ) - .await?; + loop { + info!("Gathering index data chunk {idx}"); + idx += 1; + + let (uploads, next_cursor) = + index_local_chunk(&pool, cursor, 10000).await?; + total += uploads.len(); + + if uploads.is_empty() { + info!( + "No more projects to index, indexed {total} projects after {idx} chunks" + ); + break; + } + + cursor = next_cursor; + + add_projects_batch_client( + &indices, + uploads, + all_loader_fields.clone(), + config, + ) + .await?; + } // Swap the index swap_index(config, "projects").await?;