Algolia fields filtering (#946)

* wip

* crud for algolia fields settings

* add ui form fields confiugraion

* adjust app to new config

* filter mapping with fields

* fix lang

* fix lang

.changeset/angry-ravens-enjoy.md

@@ -0,0 +1,5 @@
+---
+"saleor-app-search": minor
+---
+
+Added fields filtering form. Unused fields can be unchecked to match Algolia limits. By default every field is selected

apps/search/src/components/AlgoliaConfigurationForm.tsx

@@ -31,14 +31,14 @@ export const AlgoliaConfigurationForm = () => {
   const { isLoading: isQueryLoading, refetch: refetchConfig } =
     trpcClient.configuration.getConfig.useQuery(undefined, {
       onSuccess(data) {
-        setValue("secretKey", data?.secretKey || "");
-        setValue("appId", data?.appId || "");
-        setValue("indexNamePrefix", data?.indexNamePrefix || "");
+        setValue("secretKey", data?.appConfig?.secretKey || "");
+        setValue("appId", data?.appConfig?.appId || "");
+        setValue("indexNamePrefix", data?.appConfig?.indexNamePrefix || "");
       },
     });
 
   const { mutate: setConfig, isLoading: isMutationLoading } =
-    trpcClient.configuration.setConfig.useMutation({
+    trpcClient.configuration.setConnectionConfig.useMutation({
       onSuccess: async () => {
         await Promise.all([
           refetchConfig(),
@@ -59,6 +59,7 @@ export const AlgoliaConfigurationForm = () => {
       appId: conf.appId ?? "",
       apiKey: conf.secretKey ?? "",
       indexNamePrefix: conf.indexNamePrefix,
+      enabledKeys: [], // not required for ping but should be refactored
     });
 
     try {
@@ -85,6 +86,7 @@ export const AlgoliaConfigurationForm = () => {
               disabled={isFormDisabled}
               required
               label="Application ID"
+              /* cspell:disable-next-line */
               helperText="Usually 10 characters, e.g. XYZAAABB00"
             />
           </Box>

apps/search/src/components/AlgoliaFieldsSelectionForm.tsx

@@ -0,0 +1,80 @@
+import { Box, Checkbox, Divider, Skeleton, Button } from "@saleor/macaw-ui/next";
+import { trpcClient } from "../modules/trpc/trpc-client";
+import {
+  AlgoliaRootFields,
+  AlgoliaRootFieldsKeys,
+  AlgoliaRootFieldsLabelsMap,
+} from "../lib/algolia-fields";
+import { Controller, useForm } from "react-hook-form";
+import { useEffect } from "react";
+import { useDashboardNotification } from "@saleor/apps-shared";
+
+export const AlgoliaFieldsSelectionForm = () => {
+  const { notifySuccess } = useDashboardNotification();
+
+  const { setValue, control, handleSubmit } = useForm<Record<AlgoliaRootFields, boolean>>({});
+
+  const { data: config, isLoading } = trpcClient.configuration.getConfig.useQuery();
+  const { mutate } = trpcClient.configuration.setFieldsMappingConfig.useMutation({
+    onSuccess() {
+      notifySuccess("Success", "Algolia will be updated only with selected fields");
+    },
+  });
+
+  useEffect(() => {
+    if (config) {
+      config.fieldsMapping.enabledAlgoliaFields.forEach((field) => {
+        setValue(field as AlgoliaRootFields, true);
+      });
+    }
+  }, [config, setValue]);
+
+  if (isLoading || !config) {
+    // todo replace with Section Skeleton
+    return <Skeleton height={5} />;
+  }
+
+  return (
+    <Box>
+      <form
+        onSubmit={handleSubmit((values) => {
+          const selectedValues = Object.entries(values)
+            .filter(([key, selected]) => selected)
+            .map(([key]) => key);
+
+          mutate({
+            enabledAlgoliaFields: selectedValues,
+          });
+        })}
+      >
+        <Box padding={5}>
+          {AlgoliaRootFieldsKeys.map((field) => (
+            <Box key={field} marginBottom={5}>
+              <Controller
+                name={field}
+                control={control}
+                render={({ field: { value, onChange } }) => {
+                  return (
+                    <Checkbox
+                      onCheckedChange={(v) => {
+                        onChange(v);
+                      }}
+                      checked={value}
+                      name={field}
+                    >
+                      {AlgoliaRootFieldsLabelsMap[field]}
+                    </Checkbox>
+                  );
+                }}
+              />
+            </Box>
+          ))}
+        </Box>
+        <Divider margin={0} marginTop={5} />
+        <Box padding={5} display="flex" justifyContent="flex-end">
+          <Button type="submit">Save</Button>
+        </Box>
+      </form>
+    </Box>
+  );
+};

apps/search/src/components/ImportProductsToAlgolia.tsx

@@ -17,18 +17,19 @@ export const ImportProductsToAlgolia = () => {
   const { data: algoliaConfiguration } = trpcClient.configuration.getConfig.useQuery();
 
   const searchProvider = useMemo(() => {
-    if (!algoliaConfiguration?.appId || !algoliaConfiguration.secretKey) {
+    if (!algoliaConfiguration?.appConfig?.appId || !algoliaConfiguration.appConfig?.secretKey) {
       return null;
     }
     return new AlgoliaSearchProvider({
-      appId: algoliaConfiguration.appId,
-      apiKey: algoliaConfiguration.secretKey,
-      indexNamePrefix: algoliaConfiguration.indexNamePrefix,
+      appId: algoliaConfiguration.appConfig.appId,
+      apiKey: algoliaConfiguration.appConfig.secretKey,
+      indexNamePrefix: algoliaConfiguration.appConfig.indexNamePrefix,
+      enabledKeys: algoliaConfiguration.fieldsMapping.enabledAlgoliaFields,
     });
   }, [
-    algoliaConfiguration?.appId,
-    algoliaConfiguration?.indexNamePrefix,
-    algoliaConfiguration?.secretKey,
+    algoliaConfiguration?.appConfig?.appId,
+    algoliaConfiguration?.appConfig?.indexNamePrefix,
+    algoliaConfiguration?.appConfig?.secretKey,
   ]);
 
   const importProducts = useCallback(() => {

apps/search/src/components/IndicesSettings.tsx

@@ -8,7 +8,8 @@ export const IndicesSettings = () => {
   const { data: algoliaConfiguration } = trpcClient.configuration.getConfig.useQuery();
   const updateWebhooksMutation = useIndicesSetupMutation();
 
-  const isConfigured = algoliaConfiguration?.appId && algoliaConfiguration?.secretKey;
+  const isConfigured =
+    algoliaConfiguration?.appConfig?.appId && algoliaConfiguration?.appConfig?.secretKey;
 
   return (
     <Box>

apps/search/src/domain/WebhookActivityToggler.service.test.ts

@@ -1,6 +1,9 @@
 import { Client } from "urql";
 import { describe, it, expect, vi, beforeEach } from "vitest";
-import { IWebhooksActivityClient, WebhookActivityTogglerService } from "./WebhookActivityToggler.service";
+import {
+  IWebhooksActivityClient,
+  WebhookActivityTogglerService,
+} from "./WebhookActivityToggler.service";
 
 describe("WebhookActivityTogglerService", function () {
   let mockWebhooksClient: IWebhooksActivityClient;
@@ -11,6 +14,8 @@ describe("WebhookActivityTogglerService", function () {
       enableSingleWebhook: vi.fn(),
       disableSingleWebhook: vi.fn(),
       fetchAppWebhooksIDs: vi.fn(),
+      createWebhook: vi.fn(),
+      removeSingleWebhook: vi.fn(),
     };
 
     service = new WebhookActivityTogglerService("ID", {} as Client, {

apps/search/src/lib/algolia-fields.ts

@@ -0,0 +1,34 @@
+export type AlgoliaRootFields =
+  | "attributes"
+  | "media"
+  | "description"
+  | "descriptionPlaintext"
+  | "categories"
+  | "collections"
+  | "metadata"
+  | "variantMetadata"
+  | "otherVariants";
+
+export const AlgoliaRootFieldsLabelsMap = {
+  attributes: "Product and variant attributes",
+  categories: "Product categories (5 levels)",
+  collections: "Product collection names",
+  description: "Product description - JSON",
+  descriptionPlaintext: "Product description - plain text",
+  media: "Variant media (images and videos)",
+  metadata: "Product metadata",
+  otherVariants: "IDs of other variants of the same product",
+  variantMetadata: "Variant metadata",
+} satisfies Record<AlgoliaRootFields, string>;
+
+export const AlgoliaRootFieldsKeys = [
+  "attributes",
+  "media",
+  "description",
+  "descriptionPlaintext",
+  "categories",
+  "collections",
+  "metadata",
+  "variantMetadata",
+  "otherVariants",
+] as const;

apps/search/src/lib/algolia/algoliaSearchProvider.ts

@@ -18,6 +18,7 @@ export interface AlgoliaSearchProviderOptions {
   apiKey: string;
   indexNamePrefix?: string;
   channels?: Array<{ slug: string; currencyCode: string }>;
+  enabledKeys: string[];
 }
 
 const logger = createLogger({ name: "AlgoliaSearchProvider" });
@@ -26,13 +27,21 @@ export class AlgoliaSearchProvider implements SearchProvider {
   #algolia: SearchClient;
   #indexNamePrefix?: string | undefined;
   #indexNames: Array<string>;
+  #enabledKeys: string[];
 
-  constructor({ appId, apiKey, indexNamePrefix, channels }: AlgoliaSearchProviderOptions) {
+  constructor({
+    appId,
+    apiKey,
+    indexNamePrefix,
+    channels,
+    enabledKeys,
+  }: AlgoliaSearchProviderOptions) {
     this.#algolia = Algoliasearch(appId, apiKey);
     this.#indexNamePrefix = indexNamePrefix;
     this.#indexNames =
       channels?.map((c) => channelListingToAlgoliaIndexId({ channel: c }, this.#indexNamePrefix)) ||
       [];
+    this.#enabledKeys = enabledKeys;
   }
 
   private async saveGroupedByIndex(groupedByIndex: GroupedByIndex) {
@@ -96,6 +105,7 @@ export class AlgoliaSearchProvider implements SearchProvider {
     const groupedByIndex = groupProductsByIndexName(productsBatch, {
       visibleInListings: true,
       indexNamePrefix: this.#indexNamePrefix,
+      enabledKeys: this.#enabledKeys,
     });
 
     await this.saveGroupedByIndex(groupedByIndex);
@@ -139,6 +149,7 @@ export class AlgoliaSearchProvider implements SearchProvider {
     const groupedByIndexToSave = groupVariantByIndexName(productVariant, {
       visibleInListings: true,
       indexNamePrefix: this.#indexNamePrefix,
+      enabledKeys: this.#enabledKeys,
     });
 
     if (groupedByIndexToSave && !!Object.keys(groupedByIndexToSave).length) {
@@ -193,7 +204,12 @@ const groupVariantByIndexName = (
   {
     visibleInListings,
     indexNamePrefix,
-  }: { visibleInListings: true | false | null; indexNamePrefix: string | undefined },
+    enabledKeys,
+  }: {
+    visibleInListings: true | false | null;
+    indexNamePrefix: string | undefined;
+    enabledKeys: string[];
+  },
 ) => {
   logger.debug("Grouping variants per index name");
   if (!productVariant.channelListings) {
@@ -225,6 +241,7 @@ const groupVariantByIndexName = (
       const object = productAndVariantToAlgolia({
         variant: productVariant,
         channel: channelListing.channel.slug,
+        enabledKeys,
       });
 
       return {
@@ -246,13 +263,18 @@ const groupProductsByIndexName = (
   {
     visibleInListings,
     indexNamePrefix,
-  }: { visibleInListings: true | false | null; indexNamePrefix: string | undefined },
+    enabledKeys,
+  }: {
+    visibleInListings: true | false | null;
+    indexNamePrefix: string | undefined;
+    enabledKeys: string[];
+  },
 ) => {
   logger.debug(`groupProductsByIndexName called`);
   const batchesAndIndices = productsBatch
     .flatMap((p) => p.variants)
     .filter(isNotNil)
-    .map((p) => groupVariantByIndexName(p, { visibleInListings, indexNamePrefix }))
+    .map((p) => groupVariantByIndexName(p, { visibleInListings, indexNamePrefix, enabledKeys }))
     .filter(isNotNil)
     .flatMap((x) => Object.entries(x));
 

apps/search/src/lib/algolia/algoliaUtils.ts

@@ -6,6 +6,7 @@ import {
 import { isNotNil } from "../isNotNil";
 import { safeParseJson } from "../safe-parse-json";
 import { metadataToAlgoliaAttribute } from "./metadata-to-algolia-attribute";
+import { AlgoliaRootFields, AlgoliaRootFieldsKeys } from "../algolia-fields";
 
 type PartialChannelListing = {
   channel: {
@@ -82,9 +83,11 @@ const mapSelectedAttributesToRecord = (attr: ProductAttributesDataFragment) => {
 export function productAndVariantToAlgolia({
   variant,
   channel,
+  enabledKeys,
 }: {
   variant: ProductVariantWebhookPayloadFragment;
   channel: string;
+  enabledKeys: string[];
 }) {
   const product = variant.product;
   const attributes = {
@@ -138,7 +141,16 @@ export function productAndVariantToAlgolia({
     metadata: metadataToAlgoliaAttribute(variant.product.metadata),
     variantMetadata: metadataToAlgoliaAttribute(variant.metadata),
     otherVariants: variant.product.variants?.map((v) => v.id).filter((v) => v !== variant.id) || [],
-  };
+  } satisfies Record<AlgoliaRootFields | string, unknown>;
+
+  // todo refactor
+  AlgoliaRootFieldsKeys.forEach((field) => {
+    const enabled = enabledKeys.includes(field);
+
+    if (!enabled) {
+      delete document[field];
+    }
+  });
 
   return document;
 }

apps/search/src/modules/configuration/configuration.router.ts

@@ -2,13 +2,13 @@ import { createLogger } from "@saleor/apps-shared";
 import { TRPCError } from "@trpc/server";
 import { ChannelsDocument } from "../../../generated/graphql";
 import { WebhookActivityTogglerService } from "../../domain/WebhookActivityToggler.service";
-import { AppConfigurationFields, AppConfigurationSchema } from "./configuration";
 import { AlgoliaSearchProvider } from "../../lib/algolia/algoliaSearchProvider";
 import { createSettingsManager } from "../../lib/metadata";
 import { protectedClientProcedure } from "../trpc/protected-client-procedure";
 import { router } from "../trpc/trpc-server";
-import { fetchLegacyConfiguration } from "./legacy-configuration";
 import { AppConfigMetadataManager } from "./app-config-metadata-manager";
+import { AppConfigurationSchema, FieldsConfigSchema } from "./configuration";
+import { fetchLegacyConfiguration } from "./legacy-configuration";
 
 const logger = createLogger({ name: "configuration.router" });
 
@@ -17,7 +17,7 @@ export const configurationRouter = router({
     const settingsManager = createSettingsManager(ctx.apiClient, ctx.appId);
 
     /**
-     * Backwards compatbitility
+     * Backwards compatibility
      */
     const domain = new URL(ctx.saleorApiUrl).host;
 
@@ -39,7 +39,7 @@ export const configurationRouter = router({
       return config.getConfig();
     }
   }),
-  setConfig: protectedClientProcedure
+  setConnectionConfig: protectedClientProcedure
     .meta({ requiredClientPermissions: ["MANAGE_APPS"] })
     .input(AppConfigurationSchema)
     .mutation(async ({ input, ctx }) => {
@@ -51,6 +51,7 @@ export const configurationRouter = router({
         apiKey: input.secretKey,
         indexNamePrefix: input.indexNamePrefix,
         channels,
+        enabledKeys: [], // not required to ping algolia, but should be refactored
       });
 
       const settingsManager = createSettingsManager(ctx.apiClient, ctx.appId);
@@ -84,4 +85,17 @@ export const configurationRouter = router({
 
       return null;
     }),
+  setFieldsMappingConfig: protectedClientProcedure
+    .meta({ requiredClientPermissions: ["MANAGE_APPS"] })
+    .input(FieldsConfigSchema)
+    .mutation(async ({ ctx, input }) => {
+      const settingsManager = createSettingsManager(ctx.apiClient, ctx.appId);
+      const configManager = new AppConfigMetadataManager(settingsManager);
+
+      const config = await configManager.get(ctx.saleorApiUrl);
+
+      config.setFieldsMapping(input.enabledAlgoliaFields);
+
+      configManager.set(config, ctx.saleorApiUrl);
+    }),
 });

apps/search/src/modules/configuration/configuration.ts

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { AlgoliaRootFieldsKeys } from "../../lib/algolia-fields";
 
 export const AppConfigurationSchema = z.object({
   appId: z.string().min(3),
@@ -6,17 +7,29 @@ export const AppConfigurationSchema = z.object({
   secretKey: z.string().min(3),
 });
 
-export type AppConfigurationFields = z.infer<typeof AppConfigurationSchema>;
+export const FieldsConfigSchema = z.object({
+  enabledAlgoliaFields: z.array(z.string()),
+});
 
-export const AppConfigRootSchema = AppConfigurationSchema.nullable();
+const AppConfigRootSchema = z.object({
+  appConfig: AppConfigurationSchema.nullable(),
+  fieldsMapping: FieldsConfigSchema,
+});
+
+export type AppConfigurationFields = z.infer<typeof AppConfigurationSchema>;
 export type AppConfigRootSchemaFields = z.infer<typeof AppConfigRootSchema>;
 
 export class AppConfig {
-  private rootData: AppConfigRootSchemaFields = null;
+  private rootData: AppConfigRootSchemaFields = {
+    appConfig: null,
+    fieldsMapping: {
+      enabledAlgoliaFields: [...AlgoliaRootFieldsKeys],
+    },
+  };
 
   constructor(initialData?: AppConfigRootSchemaFields) {
     if (initialData) {
-      this.rootData = AppConfigurationSchema.parse(initialData);
+      this.rootData = AppConfigRootSchema.parse(initialData);
     }
   }
 
@@ -29,7 +42,15 @@ export class AppConfig {
   }
 
   setAlgoliaSettings(settings: AppConfigurationFields) {
-    this.rootData = AppConfigurationSchema.parse(settings);
+    this.rootData.appConfig = AppConfigurationSchema.parse(settings);
+
+    return this;
+  }
+
+  setFieldsMapping(fieldsMapping: string[]) {
+    this.rootData.fieldsMapping = {
+      enabledAlgoliaFields: z.array(z.string()).parse(fieldsMapping),
+    };
 
     return this;
   }

apps/search/src/pages/api/setup-indices.ts

@@ -7,6 +7,7 @@ import { createGraphQLClient } from "@saleor/apps-shared";
 import { Client } from "urql";
 import { ChannelsDocument } from "../../../generated/graphql";
 import { AlgoliaSearchProvider } from "../../lib/algolia/algoliaSearchProvider";
+import { AppConfigMetadataManager } from "../../modules/configuration/app-config-metadata-manager";
 
 const logger = createLogger({
   service: "setupIndicesHandler",
@@ -31,28 +32,28 @@ export const setupIndicesHandlerFactory =
     logger.debug("Fetching settings");
     const client = graphqlClientFactory(authData.saleorApiUrl, authData.token);
     const settingsManager = settingsManagerFactory(client, authData.appId);
+    const configManager = new AppConfigMetadataManager(settingsManager);
 
-    const domain = new URL(authData.saleorApiUrl).host;
-
-    const [secretKey, appId, indexNamePrefix, channelsRequest] = await Promise.all([
-      settingsManager.get("secretKey", domain),
-      settingsManager.get("appId", domain),
-      settingsManager.get("indexNamePrefix", domain),
+    const [config, channelsRequest] = await Promise.all([
+      configManager.get(authData.saleorApiUrl),
       client.query(ChannelsDocument, {}).toPromise(),
     ]);
 
-    if (!secretKey || !appId) {
-      logger.debug("Missing secretKey or appId, returning 400");
+    const configData = config.getConfig();
+
+    if (!configData.appConfig) {
+      logger.debug("Missing config, returning 400");
       return res.status(400).end();
     }
 
     const channels = channelsRequest.data?.channels || [];
 
     const algoliaClient = new AlgoliaSearchProvider({
-      appId,
-      apiKey: secretKey,
-      indexNamePrefix: indexNamePrefix,
+      appId: configData.appConfig.appId,
+      apiKey: configData.appConfig.secretKey,
+      indexNamePrefix: configData.appConfig.indexNamePrefix,
       channels,
+      enabledKeys: configData.fieldsMapping.enabledAlgoliaFields,
     });
 
     try {

apps/search/src/pages/api/webhooks-status.ts

@@ -126,7 +126,7 @@ export default createProtectedHandler(
       return new WebhookActivityTogglerService(appId, client);
     },
     algoliaSearchProviderFactory(appId, apiKey) {
-      return new AlgoliaSearchProvider({ appId, apiKey });
+      return new AlgoliaSearchProvider({ appId, apiKey, enabledKeys: [] });
     },
     graphqlClientFactory(saleorApiUrl: string, token: string) {
       return createGraphQLClient({ saleorApiUrl, token });

apps/search/src/views/configuration/configuration.view.tsx

@@ -7,6 +7,7 @@ import { MainInstructions } from "../../components/MainInstructions";
 import { WebhooksStatusInstructions } from "../../components/WebhooksStatusInstructions";
 import { TextLink } from "@saleor/apps-ui";
 import { IndicesSettings } from "../../components/IndicesSettings";
+import { AlgoliaFieldsSelectionForm } from "../../components/AlgoliaFieldsSelectionForm";
 
 const ALGOLIA_DASHBOARD_TOKENS_URL = "https://www.algolia.com/account/api-keys/all";
 
@@ -44,6 +45,22 @@ export const ConfigurationView = () => {
           </Box>
         }
       />
+      <AppSection
+        marginTop={14}
+        heading="Fields filtering"
+        mainContent={<AlgoliaFieldsSelectionForm />}
+        sideContent={
+          <Box>
+            <Text as="p" marginBottom={1.5}>
+              Decide which fields app should send with each product variant.
+            </Text>
+            <Text as="p" marginBottom={1.5}>
+              You should remove fields you do not need, to ensure Algolia limits will not be
+              exceeded.
+            </Text>
+          </Box>
+        }
+      />
       <AppSection
         includePadding
         marginTop={14}

apps/search/src/webhooks/webhook-context.ts

@@ -26,11 +26,16 @@ export const createWebhookContext = async ({ authData }: { authData: AuthData })
     throw new Error(errorMessage);
   }
 
+  if (!settings.appConfig) {
+    throw new Error("App not configured");
+  }
+
   const algoliaClient = new AlgoliaSearchProvider({
-    appId: settings.appId,
-    apiKey: settings.secretKey,
-    indexNamePrefix: settings.indexNamePrefix,
+    appId: settings.appConfig?.appId,
+    apiKey: settings.appConfig?.secretKey,
+    indexNamePrefix: settings.appConfig?.indexNamePrefix,
     channels,
+    enabledKeys: settings.fieldsMapping.enabledAlgoliaFields,
   });
 
   return {

cspell.json

@@ -44,7 +44,18 @@
     "Undiscounted",
     "Upstash",
     "urql",
-    "Vercel"
+    "Vercel",
+    "RudderStack",
+    "DataDog",
+    "Clearpay",
+    "Afterpay",
+    "Nuvo",
+    "Appstore",
+    "tRPC",
+    "Algoliasearch",
+    "tanstack",
+    "hookform",
+    "urql"
   ],
   "ignorePaths": ["node_modules", "package.json", "pnpm-lock.yaml", ".gitignore"]
 }