feat(search): 新增ES商品索引管理功能

- 添加删除下架商品索引接口
- 实现清理无效SKU索引逻辑
- 增加商品缓存生成功能
- 扩展商品搜索服务方法
-优化商品详情缓存时间配置
- 新增ES删除参数传输对象EsDeleteDTO

framework/src/main/java/cn/lili/modules/goods/serviceimpl/GoodsServiceImpl.java

@@ -273,7 +273,7 @@ public class GoodsServiceImpl extends ServiceImpl<GoodsMapper, Goods> implements
             goodsVO.setWholesaleList(wholesaleList);
         }
 
-        cache.put(CachePrefix.GOODS.getPrefix() + goodsId, goodsVO);
+        cache.put(CachePrefix.GOODS.getPrefix() + goodsId, goodsVO,7200L);
         return goodsVO;
     }
 

framework/src/main/java/cn/lili/modules/search/entity/dto/EsDeleteDTO.java

@@ -0,0 +1,34 @@
+package cn.lili.modules.search.entity.dto;
+
+import lombok.*;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * ES删除DTO
+ * 用于封装删除ES的参数
+ */
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class EsDeleteDTO {
+
+    /**
+     * 删除的索引
+     */
+    private List<String> ids;
+
+    /**
+     * 查询字段
+     */
+    private Map<String, Object> queryFields;
+
+    /**
+     * 删除的索引
+     */
+    @NonNull
+    private Class<?> clazz;
+
+}

framework/src/main/java/cn/lili/modules/search/service/EsGoodsIndexService.java

@@ -19,6 +19,18 @@ import java.util.Map;
  **/
 public interface EsGoodsIndexService {
 
+    /**
+     * 删除下架商品索引
+     */
+    Boolean deleteGoodsDown();
+
+    /**
+     * 删除不存在的索引
+     * @return
+     */
+    Boolean delSkuIndex();
+
+    Boolean goodsCache();
     /**
      * 全局索引数据初始化
      */

framework/src/main/java/cn/lili/modules/search/service/EsGoodsSearchService.java

@@ -6,6 +6,8 @@ import cn.lili.modules.search.entity.dos.EsGoodsRelatedInfo;
 import cn.lili.modules.search.entity.dto.EsGoodsSearchDTO;
 import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import org.springframework.data.elasticsearch.core.SearchPage;
+import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
+import org.springframework.data.elasticsearch.core.query.Query;
 
 import java.util.List;
 
@@ -26,6 +28,16 @@ public interface EsGoodsSearchService {
      */
     SearchPage<EsGoodsIndex> searchGoods(EsGoodsSearchDTO searchDTO, PageVO pageVo);
 
+    /**
+     * 搜索商品
+     *
+     * @param searchQuery 搜索条件
+     * @param clazz       搜索结果类
+     * @param <T>         泛型
+     * @return 搜索结果
+     */
+    <T> SearchPage<T> searchGoods(Query searchQuery, Class<T> clazz);
+
     /**
      * 商品搜索
      *
@@ -59,4 +71,13 @@ public interface EsGoodsSearchService {
      * @return 商品索引
      */
     EsGoodsIndex getEsGoodsById(String id);
+
+    /**
+     * 创建搜索条件
+     *
+     * @param searchDTO 搜索参数
+     * @param pageVo    分页参数
+     * @return 搜索条件
+     */
+    NativeSearchQueryBuilder createSearchQueryBuilder(EsGoodsSearchDTO searchDTO, PageVO pageVo);
 }

framework/src/main/java/cn/lili/modules/search/serviceimpl/EsGoodsIndexServiceImpl.java

@@ -2,6 +2,7 @@ package cn.lili.modules.search.serviceimpl;
 
 import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.map.MapUtil;
 import cn.hutool.core.text.CharSequenceUtil;
 import cn.hutool.core.thread.ThreadUtil;
 import cn.hutool.core.util.ReflectUtil;
@@ -18,16 +19,15 @@ import cn.lili.common.vo.PageVO;
 import cn.lili.elasticsearch.BaseElasticsearchService;
 import cn.lili.elasticsearch.EsSuffix;
 import cn.lili.elasticsearch.config.ElasticsearchProperties;
+import cn.lili.modules.goods.entity.dos.Goods;
 import cn.lili.modules.goods.entity.dos.GoodsSku;
 import cn.lili.modules.goods.entity.dto.GoodsParamsDTO;
+import cn.lili.modules.goods.entity.dto.GoodsSearchParams;
 import cn.lili.modules.goods.entity.dto.GoodsSkuDTO;
 import cn.lili.modules.goods.entity.enums.GoodsAuthEnum;
 import cn.lili.modules.goods.entity.enums.GoodsSalesModeEnum;
 import cn.lili.modules.goods.entity.enums.GoodsStatusEnum;
-import cn.lili.modules.goods.service.BrandService;
+import cn.lili.modules.goods.service.*;
-import cn.lili.modules.goods.service.CategoryService;
-import cn.lili.modules.goods.service.GoodsSkuService;
-import cn.lili.modules.goods.service.StoreGoodsLabelService;
 import cn.lili.modules.promotion.entity.dos.BasePromotions;
 import cn.lili.modules.promotion.entity.dos.PromotionGoods;
 import cn.lili.modules.promotion.entity.enums.PromotionsScopeTypeEnum;
@@ -38,13 +38,16 @@ import cn.lili.modules.promotion.tools.PromotionTools;
 import cn.lili.modules.search.entity.dos.CustomWords;
 import cn.lili.modules.search.entity.dos.EsGoodsAttribute;
 import cn.lili.modules.search.entity.dos.EsGoodsIndex;
+import cn.lili.modules.search.entity.dto.EsDeleteDTO;
 import cn.lili.modules.search.entity.dto.EsGoodsSearchDTO;
 import cn.lili.modules.search.repository.EsGoodsIndexRepository;
 import cn.lili.modules.search.service.CustomWordsService;
 import cn.lili.modules.search.service.EsGoodsIndexService;
 import cn.lili.modules.search.service.EsGoodsSearchService;
+import cn.lili.mybatis.util.PageUtil;
 import cn.lili.rocketmq.RocketmqSendCallbackBuilder;
 import cn.lili.rocketmq.tags.GoodsTagsEnum;
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.core.metadata.IPage;
 import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
@@ -65,10 +68,13 @@ import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.PageRequest;
+import org.springframework.data.domain.Pageable;
 import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
 import org.springframework.data.elasticsearch.core.SearchHit;
 import org.springframework.data.elasticsearch.core.SearchHits;
 import org.springframework.data.elasticsearch.core.SearchPage;
+import org.springframework.data.elasticsearch.core.query.FetchSourceFilter;
+import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
 import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
 import org.springframework.stereotype.Service;
 
@@ -111,6 +117,8 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
     @Autowired
     private GoodsSkuService goodsSkuService;
     @Autowired
+    private GoodsService goodsService;
+    @Autowired
     private BrandService brandService;
 
     @Autowired
@@ -119,6 +127,8 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
     @Autowired
     private StoreGoodsLabelService storeGoodsLabelService;
     @Autowired
+    private EsGoodsSearchService esGoodsSearchService;
+    @Autowired
     private Cache<Object> cache;
     /**
      * rocketMq
@@ -145,6 +155,126 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
         list.addAll(h);
     }
 
+    @Override
+    public Boolean deleteGoodsDown() {
+        List<Goods> goodsList = goodsService.list(new LambdaQueryWrapper<Goods>().eq(Goods::getMarketEnable, GoodsStatusEnum.DOWN.name()));
+        for (Goods goods : goodsList) {
+            this.deleteIndex(
+                    EsDeleteDTO.builder()
+                            .queryFields(MapUtil.builder(new HashMap<String, Object>()).put("goodsId", goods.getId()).build())
+                            .clazz(EsGoodsIndex.class)
+                            .build());
+        }
+        return true;
+    }
+
+    @Override
+    public Boolean delSkuIndex() {
+        PageVO pageVO = new PageVO();
+        EsGoodsSearchDTO goodsSearchParams = new EsGoodsSearchDTO();
+        log.error("开始");
+        try {
+            List<Object> searchFilters = new ArrayList<>();
+            for (int i = 1; ; i++) {
+
+                log.error("第" + i + "页");
+
+                pageVO.setPageSize(1000);
+                pageVO.setPageNumber(i);
+                pageVO.setNotConvert(true);
+                pageVO.setSort("_id");
+                pageVO.setOrder("asc");
+
+                NativeSearchQueryBuilder searchQueryBuilder = esGoodsSearchService.createSearchQueryBuilder(goodsSearchParams, pageVO);
+
+
+                searchQueryBuilder.withSourceFilter(new FetchSourceFilter(new String[]{"_id"}, null));
+
+                Pageable pageable = PageRequest.of(0, 1000);
+                //分页
+                searchQueryBuilder.withPageable(pageable);
+                NativeSearchQuery query = searchQueryBuilder.build();
+                EsGoodsSearchDTO searchDTO = new EsGoodsSearchDTO();
+                SearchPage<EsGoodsIndex> searchHits = goodsSearchService.searchGoods(query, EsGoodsIndex.class);
+
+                if (searchHits == null || searchHits.isEmpty()) {
+                    break;
+                }
+
+                List<String> idList = searchHits.getSearchHits().stream().map(SearchHit::getContent).map(EsGoodsIndex::getId).collect(Collectors.toList());
+                LambdaQueryWrapper<GoodsSku> queryWrapper = new LambdaQueryWrapper<>();
+                queryWrapper.select(GoodsSku::getId);
+                queryWrapper.in(GoodsSku::getId, idList);
+                List<GoodsSku> goodsSkus = goodsSkuService.list(queryWrapper);
+
+                idList.forEach(id -> {
+                    if (goodsSkus.stream().noneMatch(goodsSku -> goodsSku.getId().equals(id))) {
+                        log.error("[{}]不存在，进行删除", id);
+                        this.deleteIndexById(id);
+                    }
+                });
+
+
+                if (!searchHits.getSearchHits().getSearchHit(idList.size() -1).getSortValues().isEmpty()) {
+                    searchFilters = searchHits.getSearchHits().getSearchHit(idList.size() -1).getSortValues();
+                }
+
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        System.out.println("结束了");
+        return true;
+    }
+
+    @Override
+    public Boolean goodsCache() {
+        GoodsSearchParams searchParams = new GoodsSearchParams();
+        searchParams.setAuthFlag(GoodsAuthEnum.PASS.name());
+        searchParams.setMarketEnable(GoodsStatusEnum.UPPER.name());
+
+        for (int i = 1; ; i++) {
+            try {
+                IPage<Goods> pagePage = new Page<>();
+                searchParams.setPageSize(1000);
+                searchParams.setPageNumber(i);
+                pagePage = goodsService.queryByParams(searchParams);
+
+                if (pagePage == null || CollUtil.isEmpty(pagePage.getRecords())) {
+                    break;
+                }
+                for (Goods goods : pagePage.getRecords()) {
+                    cache.remove(CachePrefix.GOODS.getPrefix() + goods.getId());
+                    goodsService.getGoodsVO(goods.getId());
+                }
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        }
+
+
+        for (int i = 1; ; i++) {
+            try {
+                IPage<GoodsSkuDTO> skuIPage = new Page<>();
+                searchParams.setPageSize(1000);
+                searchParams.setPageNumber(i);
+                skuIPage = goodsSkuService.getGoodsSkuDTOByPage(PageUtil.initPage(searchParams),
+                        searchParams.queryWrapper());
+
+                if (skuIPage == null || CollUtil.isEmpty(skuIPage.getRecords())) {
+                    break;
+                }
+                for (GoodsSkuDTO goodsSkuDTO : skuIPage.getRecords()) {
+                    GoodsSku goodsSku = goodsSkuService.getById(goodsSkuDTO.getId());
+                    cache.put(GoodsSkuService.getCacheKeys(goodsSkuDTO.getId()), goodsSku,600L);
+                }
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        }
+        return true;
+    }
+
     @Override
     public void init() {
         //获取索引任务标识
@@ -494,6 +624,20 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
 
     }
 
+    /**
+     * 删除索引
+     *
+     * @param esDeleteDTO 删除索引参数
+     */
+    private void deleteIndex(EsDeleteDTO esDeleteDTO) {
+        if (esDeleteDTO.getIds() != null && !esDeleteDTO.getIds().isEmpty()) {
+            this.deleteIndexByIds(esDeleteDTO.getIds());
+        }
+        if (esDeleteDTO.getQueryFields() != null && esDeleteDTO.getQueryFields().size() > 0) {
+            this.deleteIndex(esDeleteDTO.getQueryFields());
+        }
+    }
+
     /**
      * 删除索引
      *

framework/src/main/java/cn/lili/modules/search/serviceimpl/EsGoodsSearchServiceImpl.java

@@ -41,6 +41,7 @@ import org.springframework.data.domain.Pageable;
 import org.springframework.data.elasticsearch.core.*;
 import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
 import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
+import org.springframework.data.elasticsearch.core.query.Query;
 import org.springframework.stereotype.Service;
 
 import java.util.*;
@@ -98,6 +99,12 @@ public class EsGoodsSearchServiceImpl implements EsGoodsSearchService {
         return SearchHitSupport.searchPageFor(search, searchQuery.getPageable());
     }
 
+    @Override
+    public <T> SearchPage<T> searchGoods(Query searchQuery, Class<T> clazz) {
+        SearchHits<T> search = restTemplate.search(searchQuery, clazz);
+        return SearchHitSupport.searchPageFor(search, searchQuery.getPageable());
+    }
+
     @Override
     public Page<EsGoodsIndex> searchGoodsByPage(EsGoodsSearchDTO searchDTO, PageVO pageVo) {
         // 判断商品索引是否存在
@@ -402,7 +409,8 @@ public class EsGoodsSearchServiceImpl implements EsGoodsSearchService {
      * @param pageVo    分页参数
      * @return es搜索builder
      */
-    private NativeSearchQueryBuilder createSearchQueryBuilder(EsGoodsSearchDTO searchDTO, PageVO pageVo) {
+    @Override
+    public NativeSearchQueryBuilder createSearchQueryBuilder(EsGoodsSearchDTO searchDTO, PageVO pageVo) {
         NativeSearchQueryBuilder nativeSearchQueryBuilder = new NativeSearchQueryBuilder();
         if (pageVo != null) {
             int pageNumber = pageVo.getPageNumber() - 1;

manager-api/src/main/java/cn/lili/controller/other/ElasticsearchController.java

@@ -3,8 +3,11 @@ package cn.lili.controller.other;
 import cn.lili.common.enums.ResultUtil;
 import cn.lili.common.vo.ResultMessage;
 import cn.lili.modules.search.service.EsGoodsIndexService;
+import cn.lili.modules.system.aspect.annotation.SystemLogPoint;
 import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.DeleteMapping;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
@@ -36,4 +39,30 @@ public class ElasticsearchController {
     public ResultMessage<Map<String, Long>> getProgress() {
         return ResultUtil.data(esGoodsIndexService.getProgress());
     }
+
+
+    @ApiOperation(value = "ES删除下架的商品")
+    @GetMapping(value = "/deleteGoodsDown")
+    @SystemLogPoint(description = "ES删除下架的商品", customerLog = "")
+    public ResultMessage<Object> deleteGoodsDown() {
+
+        esGoodsIndexService.deleteGoodsDown();
+        return ResultUtil.success();
+    }
+
+    @ApiOperation(value = "删除不存在的索引")
+    @GetMapping(value = "/delSkuIndex")
+    @SystemLogPoint(description = "删除不存在的索引", customerLog = "")
+    public ResultMessage<Object> delSkuIndex() {
+        esGoodsIndexService.delSkuIndex();
+        return ResultUtil.success();
+    }
+
+    @ApiOperation(value = "生成所有商品的缓存")
+    @GetMapping(value = "/cache")
+    @SystemLogPoint(description = "生成所有商品的缓存")
+    public ResultMessage<Object> cache() {
+        esGoodsIndexService.goodsCache();
+        return ResultUtil.success();
+    }
 }