Merge pull request #1990 from mintlayer/feature/api-server-ticker-partial-matches

API Server, add partial matching for token ticker

Author: OBorce

api-server/api-server-common/src/storage/impls/in_memory/mod.rs

@@ -714,19 +714,26 @@ impl ApiServerInMemoryStorage {
         &self,
         len: u32,
         offset: u64,
-        ticker: &[u8],
+        ticker: &str,
     ) -> Result<Vec<TokenId>, ApiServerStorageError> {
+        let lowercased_ticker = ticker.to_ascii_lowercase();
+
         Ok(self
             .fungible_token_data
             .iter()
             .filter_map(|(key, value)| {
-                (value.values().last().expect("not empty").token_ticker == ticker).then_some(key)
+                let token_ticker = &value.values().last().expect("not empty").token_ticker;
+                let lowercased_token_ticker =
+                    String::from_utf8_lossy(token_ticker).to_ascii_lowercase();
+                (lowercased_token_ticker.contains(&lowercased_ticker)).then_some(key)
             })
             .chain(self.nft_token_issuances.iter().filter_map(|(key, value)| {
                 let value_ticker = match &value.values().last().expect("not empty").nft {
                     NftIssuance::V0(data) => data.metadata.ticker(),
                 };
-                (value_ticker == ticker).then_some(key)
+                let lowercased_value_ticker =
+                    String::from_utf8_lossy(value_ticker).to_ascii_lowercase();
+                (lowercased_value_ticker.contains(&lowercased_ticker)).then_some(key)
             }))
             .skip(offset as usize)
             .take(len as usize)

api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs

@@ -268,7 +268,7 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRo<'_> {
         &self,
         len: u32,
         offset: u64,
-        ticker: &[u8],
+        ticker: &str,
     ) -> Result<Vec<TokenId>, ApiServerStorageError> {
         self.transaction.get_token_ids_by_ticker(len, offset, ticker)
     }

api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs

@@ -527,7 +527,7 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRw<'_> {
         &self,
         len: u32,
         offset: u64,
-        ticker: &[u8],
+        ticker: &str,
     ) -> Result<Vec<TokenId>, ApiServerStorageError> {
         self.transaction.get_token_ids_by_ticker(len, offset, ticker)
     }

api-server/api-server-common/src/storage/impls/mod.rs

@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub const CURRENT_STORAGE_VERSION: u32 = 22;
+pub const CURRENT_STORAGE_VERSION: u32 = 23;
 
 pub mod in_memory;
 pub mod postgres;

api-server/api-server-common/src/storage/impls/postgres/queries.rs

@@ -830,7 +830,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> {
             "CREATE TABLE ml.fungible_token (
                     token_id bytea NOT NULL,
                     block_height bigint NOT NULL,
-                    ticker bytea NOT NULL,
+                    ticker TEXT NOT NULL,
                     authority TEXT NOT NULL,
                     issuance bytea NOT NULL,
                     PRIMARY KEY (token_id, block_height)
@@ -840,7 +840,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> {
 
         // index when searching for token tickers
         self.just_execute(
-            "CREATE INDEX fungible_token_ticker_index ON ml.fungible_token USING HASH (ticker);",
+            "CREATE INDEX fungible_token_ticker_case_insensitive_idx ON ml.fungible_token (ticker text_pattern_ops);"
         )
         .await?;
 
@@ -854,7 +854,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> {
             "CREATE TABLE ml.nft_issuance (
                     nft_id bytea NOT NULL,
                     block_height bigint NOT NULL,
-                    ticker bytea NOT NULL,
+                    ticker TEXT NOT NULL,
                     issuance bytea NOT NULL,
                     owner bytea,
                     PRIMARY KEY (nft_id, block_height)
@@ -864,7 +864,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> {
 
         // index when searching for token tickers
         self.just_execute(
-            "CREATE INDEX nft_token_ticker_index ON ml.nft_issuance USING HASH (ticker);",
+            "CREATE INDEX nft_token_ticker_case_insensitive_idx ON ml.nft_issuance (ticker text_pattern_ops);"
         )
         .await?;
 
@@ -2215,7 +2215,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> {
                     &token_id.encode(),
                     &height,
                     &issuance.encode(),
-                    &issuance.token_ticker,
+                    &String::from_utf8(issuance.token_ticker).expect("Ticker is valid UTF-8 string"),
                     &authority.into_string(),
                 ],
             )
@@ -2254,7 +2254,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> {
                     &token_id.encode(),
                     &height,
                     &issuance.encode(),
-                    &issuance.token_ticker,
+                    &String::from_utf8(issuance.token_ticker).expect("Ticker is valid UTF-8 string"),
                     &authority.into_string(),
                 ],
             )
@@ -2397,34 +2397,36 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> {
         &self,
         len: u32,
         offset: u64,
-        ticker: &[u8],
+        ticker: &str,
     ) -> Result<Vec<TokenId>, ApiServerStorageError> {
         let len = len as i64;
         let offset = offset as i64;
+        let escaped_ticker = escape_for_like(ticker);
+        let ticker_patern = format!("%{escaped_ticker}%");
         self.tx
             .query(
                 r#"
                 WITH count_tokens AS (
-                    SELECT count(token_id) FROM ml.fungible_token WHERE ticker = $3
+                    SELECT count(token_id) FROM ml.fungible_token WHERE ticker ILIKE $3
                 )
                 (SELECT token_id
                  FROM ml.fungible_token
-                 WHERE ticker = $3
+                 WHERE ticker ILIKE $3
                  ORDER BY token_id
                  OFFSET $1
                  LIMIT $2)
                 UNION ALL
                 (SELECT nft_id
                  FROM ml.nft_issuance
-                 WHERE ticker = $3
+                 WHERE ticker ILIKE $3
                  ORDER BY nft_id
                  OFFSET GREATEST($1 - (SELECT * FROM count_tokens), 0)
                  LIMIT CASE
                        WHEN ($1 - (SELECT * FROM count_tokens) >= -$2)
                            THEN ($2 + $1 - (SELECT * FROM count_tokens))
                        ELSE 0 END);
             "#,
-                &[&offset, &len, &ticker],
+                &[&offset, &len, &ticker_patern],
             )
             .await
             .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?
@@ -2626,7 +2628,13 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> {
         self.tx
             .execute(
                 "INSERT INTO ml.nft_issuance (nft_id, block_height, issuance, ticker, owner) VALUES ($1, $2, $3, $4, $5);",
-                &[&token_id.encode(), &height, &issuance.encode(), ticker, &owner.encode()],
+                &[
+                    &token_id.encode(),
+                    &height,
+                    &issuance.encode(),
+                    &String::from_utf8(ticker.clone()).expect("Ticker is valid UTF-8 string"),
+                    &owner.encode()
+                ],
             )
             .await
             .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?;
@@ -2977,3 +2985,20 @@ fn amount_to_str(amount: Amount) -> String {
     }
     amount_str
 }
+
+// Escapes a string for use in a SQL LIKE clause
+fn escape_for_like(input: &str) -> String {
+    let mut escaped = String::with_capacity(input.len());
+    for c in input.chars() {
+        match c {
+            '%' | '_' | '\\' => {
+                escaped.push('\\');
+                escaped.push(c);
+            }
+            _ => {
+                escaped.push(c);
+            }
+        }
+    }
+    escaped
+}

api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs

@@ -378,7 +378,7 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRo<'_> {
         &self,
         len: u32,
         offset: u64,
-        ticker: &[u8],
+        ticker: &str,
     ) -> Result<Vec<TokenId>, ApiServerStorageError> {
         let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR));
         let res = conn.get_token_ids_by_ticker(len, offset, ticker).await?;

api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs

@@ -714,7 +714,7 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRw<'_> {
         &self,
         len: u32,
         offset: u64,
-        ticker: &[u8],
+        ticker: &str,
     ) -> Result<Vec<TokenId>, ApiServerStorageError> {
         let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR));
         let res = conn.get_token_ids_by_ticker(len, offset, ticker).await?;

api-server/api-server-common/src/storage/storage_api/mod.rs

@@ -754,7 +754,7 @@ pub trait ApiServerStorageRead: Sync {
         &self,
         len: u32,
         offset: u64,
-        ticker: &[u8],
+        ticker: &str,
     ) -> Result<Vec<TokenId>, ApiServerStorageError>;
 
     async fn get_statistic(

api-server/storage-test-suite/src/basic.rs

@@ -1565,9 +1565,9 @@ where
         let (_, pk) = PrivateKey::new_from_rng(&mut rng, KeyKind::Secp256k1Schnorr);
         let random_destination = Destination::PublicKeyHash(PublicKeyHash::from(&pk));
 
-        let token_ticker = "XXXX".as_bytes().to_vec();
+        let token_ticker = String::from("AB\\CD");
         let token_data = FungibleTokenData {
-            token_ticker: token_ticker.clone(),
+            token_ticker: token_ticker.as_bytes().to_vec().clone(),
             number_of_decimals: rng.gen_range(1..18),
             metadata_uri: "http://uri".as_bytes().to_vec(),
             circulating_supply: Amount::ZERO,
@@ -1602,7 +1602,7 @@ where
                 creator: None,
                 name: "Name".as_bytes().to_vec(),
                 description: "SomeNFT".as_bytes().to_vec(),
-                ticker: token_ticker.clone(),
+                ticker: token_ticker.as_bytes().to_vec().clone(),
                 icon_uri: DataOrNoVec::from(None),
                 additional_metadata_uri: DataOrNoVec::from(None),
                 media_uri: DataOrNoVec::from(None),
@@ -1629,24 +1629,20 @@ where
             .unwrap();
 
         // will return all token and nft ids
-        let ids = db_tx.get_token_ids(6, 0).await.unwrap();
-        assert!(ids.contains(&random_token_id1));
-        assert!(ids.contains(&random_token_id2));
-        assert!(ids.contains(&random_token_id3));
+        let all_ids = db_tx.get_token_ids(6, 0).await.unwrap();
+        assert!(all_ids.contains(&random_token_id1));
+        assert!(all_ids.contains(&random_token_id2));
+        assert!(all_ids.contains(&random_token_id3));
 
-        assert!(ids.contains(&random_token_id4));
-        assert!(ids.contains(&random_token_id5));
-        assert!(ids.contains(&random_token_id6));
+        assert!(all_ids.contains(&random_token_id4));
+        assert!(all_ids.contains(&random_token_id5));
+        assert!(all_ids.contains(&random_token_id6));
 
         // will return all token and nft ids
         let ids = db_tx.get_token_ids_by_ticker(6, 0, &token_ticker).await.unwrap();
-        assert!(ids.contains(&random_token_id1));
-        assert!(ids.contains(&random_token_id2));
-        assert!(ids.contains(&random_token_id3));
-
-        assert!(ids.contains(&random_token_id4));
-        assert!(ids.contains(&random_token_id5));
-        assert!(ids.contains(&random_token_id6));
+        for id in &all_ids {
+            assert!(ids.contains(id));
+        }
 
         // will return the tokens first
         let ids = db_tx.get_token_ids(3, 0).await.unwrap();
@@ -1672,8 +1668,34 @@ where
         assert!(ids.contains(&random_token_id5));
         assert!(ids.contains(&random_token_id6));
 
-        let ids = db_tx.get_token_ids_by_ticker(0, 6, "NOT_FOUND".as_bytes()).await.unwrap();
+        let ids = db_tx.get_token_ids_by_ticker(0, 6, "NOT_FOUND").await.unwrap();
         assert!(ids.is_empty());
+
+        // will return all token and nft ids for partial match
+        for partial_ticker in get_all_substrings(&token_ticker) {
+            let ids = db_tx.get_token_ids_by_ticker(6, 0, partial_ticker).await.unwrap();
+            for id in &all_ids {
+                assert!(ids.contains(id));
+            }
+
+            // check lowercase as well
+            let lowercase_partial_ticker = partial_ticker.to_ascii_lowercase();
+            let ids2 =
+                db_tx.get_token_ids_by_ticker(6, 0, &lowercase_partial_ticker).await.unwrap();
+            assert_eq!(ids, ids2);
+        }
+
+        // Try out patterns inside the ticker string, they should be escaped and not work
+        let ids = db_tx.get_token_ids_by_ticker(6, 0, "A%D").await.unwrap();
+        assert!(ids.is_empty());
+
+        let ids = db_tx.get_token_ids_by_ticker(6, 0, "A_").await.unwrap();
+        assert!(ids.is_empty());
+
+        for c in ['*', '+', '?'] {
+            let ids = db_tx.get_token_ids_by_ticker(6, 0, &format!("A{c}")).await.unwrap();
+            assert!(ids.is_empty());
+        }
     }
 
     // test coin and token statistics
@@ -2006,3 +2028,13 @@ where
     ]
     .into_iter()
 }
+
+fn get_all_substrings(s: &str) -> Vec<&str> {
+    let mut substrings = Vec::new();
+    for i in 0..s.len() {
+        for j in i..s.len() {
+            substrings.push(s.get(i..=j).unwrap());
+        }
+    }
+    substrings
+}

api-server/web-server/src/api/v2.rs

@@ -1383,11 +1383,7 @@ pub async fn token_ids_by_ticker<T: ApiServerStorage>(
             logging::log::error!("internal error: {e}");
             ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError)
         })?
-        .get_token_ids_by_ticker(
-            offset_and_items.items,
-            offset_and_items.offset,
-            ticker.as_bytes(),
-        )
+        .get_token_ids_by_ticker(offset_and_items.items, offset_and_items.offset, &ticker)
         .await
         .map_err(|e| {
             logging::log::error!("internal error: {e}");