使用 Spring Boot、Google Cloud Vertex AI 和 Gemini 模型进行基于图像的产品搜索

<!-- config version for dependency-->
<properties>
    <spring-cloud-gcp.version>5.1.2</spring-cloud-gcp.version>
    <google-cloud-bom.version>26.32.0</google-cloud-bom.version>
</properties>

<!-- in your dependencymanagement, please add 2 dependencies below -->
<dependencymanagement>
  <dependencies>
      <dependency>
          <groupid>com.google.cloud</groupid>
          <artifactid>spring-cloud-gcp-dependencies</artifactid>
          <version>${spring-cloud-gcp.version}</version>
          <type>pom</type>
          <scope>import</scope>
      </dependency>

      <dependency>
          <groupid>com.google.cloud</groupid>
          <artifactid>libraries-bom</artifactid>
          <version>${google-cloud-bom.version}</version>
          <type>pom</type>
          <scope>import</scope>
      </dependency>
  </dependencies>
</dependencymanagement>

<!-- in your tab dependencies, please add the dependency below -->
<dependencies>
  <dependency>
      <groupid>com.google.cloud</groupid>
      <artifactid>google-cloud-vertexai</artifactid>
  </dependency>
</dependencies>

import com.google.cloud.vertexai.vertexai;
import com.google.cloud.vertexai.generativeai.generativemodel;
import org.springframework.context.annotation.bean;
import org.springframework.context.annotation.configuration;

@configuration(proxybeanmethods = false)
public class geminiconfig {

    private static final string model_name = "gemini-1.5-flash";
    private static final string location = "asia-southeast1";
    private static final string project_id = "yasmini";

    @bean
    public vertexai vertexai() {
        return new vertexai(project_id, location);
    }

    @bean
    public generativemodel getmodel(vertexai vertexai) {
        return new generativemodel(model_name, vertexai);
    }
}

import com.fasterxml.jackson.core.jsonprocessingexception;
import com.fasterxml.jackson.databind.objectmapper;
import com.google.cloud.vertexai.api.content;
import com.google.cloud.vertexai.api.generatecontentresponse;
import com.google.cloud.vertexai.api.part;
import com.google.cloud.vertexai.generativeai.*;
import com.learning.yasminishop.common.entity.product;
import com.learning.yasminishop.common.exception.appexception;
import com.learning.yasminishop.common.exception.errorcode;
import com.learning.yasminishop.product.productrepository;
import com.learning.yasminishop.product.dto.response.productresponse;
import com.learning.yasminishop.product.mapper.productmapper;
import lombok.requiredargsconstructor;
import lombok.extern.slf4j.slf4j;
import org.springframework.stereotype.service;
import org.springframework.transaction.annotation.transactional;
import org.springframework.web.multipart.multipartfile;

import java.util.hashset;
import java.util.list;
import java.util.objects;
import java.util.set;

@service
@requiredargsconstructor
@slf4j
@transactional(readonly = true)
public class yasminiaiservice {

    private final generativemodel generativemodel;
    private final productrepository productrepository;

    private final productmapper productmapper;


    public list<productresponse> findcarbyimage(multipartfile file){
        try {
            var prompt = "extract the name car to a list keyword and output them in json. if you don't find any information about the car, please output the list empty.\nexample response: [\"rolls\", \"royce\", \"wraith\"]";
            var content = this.generativemodel.generatecontent(
                    contentmaker.frommultimodaldata(
                            partmaker.frommimetypeanddata(objects.requirenonnull(file.getcontenttype()), file.getbytes()),
                            prompt
                    )
            );

            string jsoncontent = responsehandler.gettext(content);
            log.info("extracted keywords from image: {}", jsoncontent);
            list<string> keywords = convertjsontolist(jsoncontent).stream()
                    .map(string::tolowercase)
                    .tolist();

            set<product> results = new hashset<>();
            for (string keyword : keywords) {
                list<product> products = productrepository.searchbykeyword(keyword);
                results.addall(products);
            }

            return results.stream()
                    .map(productmapper::toproductresponse)
                    .tolist();

        } catch (exception e) {
            log.error("error finding car by image", e);
            return list.of();
        }
    }

    private list<string> convertjsontolist(string markdown) throws jsonprocessingexception {
        objectmapper objectmapper = new objectmapper();
        string parsejson = markdown;
        if(markdown.contains("```

json")){
            parsejson = extractjsonfrommarkdown(markdown);
        }
        return objectmapper.readvalue(parsejson, list.class);
    }

    private string extractjsonfrommarkdown(string markdown) {
        return markdown.replace("

```json\n", "").replace("\n```

", "");
    }
}

import com.learning.yasminishop.product.dto.response.productresponse;
import lombok.requiredargsconstructor;
import lombok.extern.slf4j.slf4j;
import org.springframework.security.access.prepost.preauthorize;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.multipartfile;

import java.util.list;

@restcontroller
@requestmapping("/ai")
@requiredargsconstructor
@slf4j
public class yasminiaicontroller {

    private final yasminiaiservice yasminiaiservice;


    @postmapping
    public list<productresponse> findcar(@requestparam("file") multipartfile file) {

        var response = yasminiaiservice.findcarbyimage(file);
        return response;
    }
}

gcloud auth login