Configuration
JavaScript Transformer Plugins

Introduction

The JavaScript Transformer plugins provide a flexible scripting engine designed to customize the Semantic Pipeline. By leveraging this engine, users can inject custom logic into specific stages of the Semantic Pipeline using ECMAScript 2021.

The framework consists of three specialized plugins:

• Item Transformer
• Post Filter Transformer
• Query Expression Transformer

The scripting engine supports calls to external services, such as LLM endpoints, allowing for the dynamic enrichment of data and behaviour via simple prompts.

Configuration

Item Transformer

Configuration of the Item Transformer can be done with the following steps:

Add the service “ItemTransformationServicePlugin.JavaScriptItemTransformer” to the Index configuration

Configure the “JavaScript Code” property

Sample JavaScript Code for Item Transformer

const Item = Java.type('mindbreeze.common.TypesProtos.Item');

const ItemTransformationResponse = Java.type('mindbreeze.processing.ItemTransformationServiceProtos.ItemTransformationResponse');

const ValueHelper = Java.type('com.mindbreeze.enterprisesearch.mesapi.protobuf.sdk.common.ValueHelper');

const Logger = Java.type('org.apache.log4j.Logger');

function transform(controller, request, done) {

    const item = Item.newBuilder(request.getItem());

    const result = myHelper(item);

    const responseBuilder = ItemTransformationResponse.newBuilder();

    responseBuilder.setItem(result);

    done.run(responseBuilder.build());

}

function myHelper(result) {

  const logger = Logger.getLogger("com.mindbreeze.javascriptitemtransformer.JavaScriptItemTransformationService");

  let key = result.getHeader().getKey();

  let category = result.getHeader().getCategory();

  logger.info("JS ItemTransformer.transform for item: " + category + "/" + key);

  for (let i = 0; i < result.getPropertyCount(); i++) {

const property = result.getProperty(i);

//logger.info("(" + i + ") found property with name: " + property.getName());

if (property.getName() === "title") {

for (let j = 0; j < property.getValueCount(); j++) {

const value = property.getValue(j);

title = value.getStringValue() || "";

//logger.info("title: " + title);

}

}

}

const myTitleProperty = TypesProtos.NamedValue.newBuilder().setName("myTitle");

myTitleProperty.addValue(ValueHelper.newBuilder("MY___" + title));

logger.info("myTitle: " + "MY___" + title);

result.addProperty(myTitleProperty);

    return result;

}

Post Filter Transformer

Configuration of the Post Filter Transformer can be done with the following steps:

Add the PostFilterPlugin.JavaScriptPostFilterTransformationService to the Filter Service configuration

Configure the “JavaScript Code Path” property

Sample for Post Filter Transformer

const Logger = Java.type('org.apache.log4j.Logger');

const FilterAndIndexRequest =  Java.type('mindbreeze.filter.FilterServiceProtos.FilterAndIndexRequest');

const NamedValue = Java.type('mindbreeze.common.TypesProtos.NamedValue');

const ValueHelper = Java.type('com.mindbreeze.enterprisesearch.mesapi.protobuf.sdk.common.ValueHelper');

const MetadataOrBuilder = Java.type('mindbreeze.common.MetadataProtos.MetadataOrBuilder');

const logger = Logger.getLogger("com.mindbreeze.javascript.JavaScriptPostFilterTransformationService");

function transform(controller, request, done) {

    try {

        const responseBuilder = FilterAndIndexRequest.newBuilder(request);

        const metadataBuilder = responseBuilder.getContentBuilder().getMetadataBuilder();

        const namedValueBuilder = NamedValue.newBuilder();

        namedValueBuilder.setName("postfiltertest");

        namedValueBuilder.addValue(ValueHelper.newBuilder("Test-success"));

        metadataBuilder.addMetadatum(namedValueBuilder);

        logger.info("Adding Metadata: " + namedValueBuilder);

        done.run(responseBuilder.build());

    } catch (e) {

        logger.error("Error in JS Post Filter Code: " + e);

    }

}

Query Expression Transformer

Configuration of the Query Expression Transformer can be done with the following steps:

Add the JavaScriptQueryExprTransformationService to the Index Service configuration

Configure the “JavaScript Code Path” property

Sample for Query Expression Transformer

const QueryExprTransformationResponse = Java.type('mindbreeze.query.QueryExprTransformationServiceProtos.QueryExprTransformationResponse');

const QueryExprProtos = Java.type('mindbreeze.query.QueryExprProtos');

const Logger = Java.type('org.apache.log4j.Logger');

const logger = Logger.getLogger("com.mindbreeze.javascript.JavaScriptQueryExprTransformationService");

const metadataToRewrite = ["SearchableMetadata"];

function transformQueryExpr(queryExpr, options) {

  options = options || {};

  const builder = QueryExprProtos.QueryExpr.newBuilder(queryExpr);

  switch (queryExpr.getKind().toString()) {

    case 'EXPR_NOT':

      const notExprBuilder = QueryExprProtos.QueryExpr.Not.newBuilder(queryExpr.getNotExpr());

      notExprBuilder.setInitialExpr(transformQueryExpr(notExprBuilder.getInitialExpr(), options));

      for (let i=0; i < notExprBuilder.getExcludedExprCount(); i++) {

        notExprBuilder.setExcludedExpr(i, transformQueryExpr(notExprBuilder.getExcludedExpr(i), options));

      }

      builder.setNotExpr(notExprBuilder);

      break;

    case 'EXPR_AND':

      const andExprBuilder = QueryExprProtos.QueryExpr.And.newBuilder(queryExpr.getAndExpr());

      for (let i=0; i < andExprBuilder.getExprCount(); i++) {

        andExprBuilder.setExpr(i, transformQueryExpr(andExprBuilder.getExpr(i), options));

      }

      builder.setAndExpr(andExprBuilder);

      break;

    case 'EXPR_OR':

      const orExprBuilder = QueryExprProtos.QueryExpr.Or.newBuilder(queryExpr.getOrExpr());

      for (let i=0; i < orExprBuilder.getExprCount(); i++) {

        orExprBuilder.setExpr(i, transformQueryExpr(orExprBuilder.getExpr(i), options));

      }

      builder.setOrExpr(orExprBuilder);

      break;

    case 'EXPR_LABELED':

      const namedExprBuilder = QueryExprProtos.QueryExpr.Labeled.newBuilder(queryExpr.getNamedExpr());

      if (metadataToRewrite.indexOf(namedExprBuilder.getLabel()) >= 0) {

        namedExprBuilder.setExpr(transformQueryExpr(namedExprBuilder.getExpr(), { rewriteToRegex: true }));

      }

      builder.setNamedExpr(namedExprBuilder);

      break;

    case 'EXPR_ALL':

      if (options.rewriteToRegex) {

        return QueryExprProtos.QueryExpr.newBuilder()

              .setKind(QueryExprProtos.QueryExpr.Kind.EXPR_REGEX_PATTERN)

              .setRegexPattern(".+");

      }

      break;

    case 'EXPR_TERM':

      if (options.rewriteToRegex) {

        return QueryExprProtos.QueryExpr.newBuilder()

              .setKind(QueryExprProtos.QueryExpr.Kind.EXPR_REGEX_PATTERN)

              .setRegexPattern("\\b\\Q" + queryExpr.getTerm() + "\\E");

      } else {

        return queryExpr;

              return QueryExprProtos.QueryExpr.newBuilder()

                .setKind(QueryExprProtos.QueryExpr.Kind.EXPR_OR)

                .setOrExpr(

                  QueryExprProtos.QueryExpr.Or.newBuilder()

                    .addExpr(queryExpr)

                  )

      }

      break;

    case 'EXPR_TERMS':

      if (options.rewriteToRegex) {

      let termsExprBuilder = QueryExprProtos.QueryExpr.And.newBuilder();

      for (let i=0; i<queryExpr.getTermsExpr().getElementCount(); i++) {

        const term = queryExpr.getTermsExpr().getElement(i).getText();

        switch (queryExpr.getTermsExpr().getElement(i).getKind().toString()) {

          case "ELEMENT_PHRASE":

            termsExprBuilder.addExpr(

              transformQueryExpr(

                QueryExprProtos.QueryExpr.newBuilder()

                  .setKind(QueryExprProtos.QueryExpr.Kind.EXPR_QUOTED_TERM)

                  .setQuotedTerm(term)

                  .build()

              )

            );

            break;

          case "ELEMENT_TERM":

            termsExprBuilder.addExpr(

              transformQueryExpr(

                QueryExprProtos.QueryExpr.newBuilder()

                  .setKind(QueryExprProtos.QueryExpr.Kind.EXPR_TERM)

                  .setTerm(term)

                  .build()

              )

            );

            break;

          default:

          logger.warn("terms expr type '" + queryExpr.getTermsExpr().getElement(i).getKind().toString() + "' not handled");

          break;

        }

      }

      return QueryExprProtos.QueryExpr.newBuilder()

                    .setKind(QueryExprProtos.QueryExpr.Kind.EXPR_AND)

                    .setAndExpr(termsExprBuilder)

      return QueryExprProtos.QueryExpr.newBuilder()

            .setKind(QueryExprProtos.QueryExpr.Kind.EXPR_OR)

            .setOrExpr(

              QueryExprProtos.QueryExpr.Or.newBuilder()

                .addExpr(queryExpr)

                .addExpr(

                  QueryExprProtos.QueryExpr.newBuilder()

                    .setKind(QueryExprProtos.QueryExpr.Kind.EXPR_AND)

                    .setAndExpr(termsExprBuilder)

                )

            )

      } else {

        return queryExpr;

      }

      break;

    case 'EXPR_QUOTED_TERM':

      if (options.rewriteToRegex) {

        return QueryExprProtos.QueryExpr.newBuilder()

              .setKind(QueryExprProtos.QueryExpr.Kind.EXPR_REGEX_PATTERN)

              .setRegexPattern("\\b\\Q" + queryExpr.getQuotedTerm() + "\\E\\b");

      } else {

              return QueryExprProtos.QueryExpr.newBuilder()

                .setKind(QueryExprProtos.QueryExpr.Kind.EXPR_OR)

                .setOrExpr(

                  QueryExprProtos.QueryExpr.Or.newBuilder()

                    .addExpr(queryExpr)

                    .addExpr(

                      QueryExprProtos.QueryExpr.newBuilder()

                        .setKind(QueryExprProtos.QueryExpr.Kind.EXPR_LABELED)

                        .setNamedExpr(

                          QueryExprProtos.QueryExpr.Labeled.newBuilder()

                            .setLabel("SearchableMetadata")

                            .setExpr(

                              QueryExprProtos.QueryExpr.newBuilder()

                                .setKind(QueryExprProtos.QueryExpr.Kind.EXPR_REGEX_PATTERN)

                                .setRegexPattern("\\b\\Q" + queryExpr.getQuotedTerm() + "\\E\\b")

                            )

                        )

                    )

                  )

      }

      break;

    default:

      logger.warn("query expr type '" + queryExpr.getKind().toString() + "' not handled");

      break;

  }

  return builder.build();

}

function transform(controller, request, done) {

    try {

        const responseBuilder = QueryExprTransformationResponse.newBuilder();

        const transformedQueryExpr = transformQueryExpr(request.getQueryExpr());

        responseBuilder.setQueryExpr(transformedQueryExpr);

        response = responseBuilder.build();

        done.run(response);

    } catch (e) {

        logger.error("error: " + e);

    }

}