sansenhoshi/weather-data
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

更新redis的使用和数据汇总性能优化

Browse Source
This commit is contained in:
sansenhoshi
2026-06-24 13:49:57 +08:00
parent 0b4860f46e
commit 4105b901f1
16 changed files with 662 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
system-admin/src/main/java/com/weather/modules/weather/dailydata/WeatherDataImportManager.java
+11
View File
@@ -9,6 +9,8 @@ import com.weather.modules.weather.dailydata.vo.WeatherExcelVO;
import com.weather.modules.security.user.SecurityUser;
import com.weather.modules.security.user.UserContextHolder;
import com.weather.modules.security.user.UserDetail;
import com.weather.common.redis.RedisKeys;
import com.weather.common.redis.RedisUtils;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@@ -27,6 +29,7 @@ import java.util.concurrent.ConcurrentHashMap;
public class WeatherDataImportManager {
private final WeatherDailyDataService weatherDailyDataService;
private final RedisUtils redisUtils;
private final Map<String, ImportProgress> tasks = new ConcurrentHashMap<>();
public String submitImport(MultipartFile file) throws IOException {
@@ -82,6 +85,14 @@ public class WeatherDataImportManager {
progress.setStatus("COMPLETED");
log.info("导入任务 {} 完成，共导入 {} 行", taskId, progress.getProcessedRows().get());
// 导入完成后清除天气汇总缓存，等待下次定时任务刷新
try {
redisUtils.deleteByPattern(RedisKeys.getWeatherSummarizePattern());
log.info("已清除天气汇总缓存");
} catch (Exception e) {
log.warn("清除天气汇总缓存失败", e);
}
}
public ImportProgress getProgress(String taskId) {
system-admin/src/main/java/com/weather/modules/weather/dailydata/dao/WeatherDailyDataDao.java
+16 -1
View File
@@ -1,5 +1,6 @@
package com.weather.modules.weather.dailydata.dao;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Constants;
import com.weather.common.dao.BaseDao;
@@ -18,7 +19,21 @@ import java.util.List;
@Mapper
public interface WeatherDailyDataDao extends BaseDao<WeatherDailyDataEntity> {
List<WeatherDailyDataEntity> selectSummarizeList(@Param(Constants.WRAPPER) QueryWrapper<WeatherDailyDataEntity> wrapper);
List<WeatherDailyDataEntity> selectSummarizeList(@Param(Constants.WRAPPER) LambdaQueryWrapper<WeatherDailyDataEntity> wrapper);
/**
* 按站点 + 月-日 + 年份范围查询汇总数据
* @param stationIds 站点 ID 列表
* @param month 月份 1-12
* @param day 日 1-31
* @param startYear 起始年份
* @param endYear 结束年份
*/
List<WeatherDailyDataEntity> selectSummarizeByMonthDay(@Param("stationIds") List<Long> stationIds,
@Param("month") int month,
@Param("day") int day,
@Param("startYear") int startYear,
@Param("endYear") int endYear);
int insertBatchMultiRow(@Param("list") List<WeatherDailyDataEntity> list);
}
system-admin/src/main/java/com/weather/modules/weather/dailydata/service/impl/WeatherDailyDataServiceImpl.java
+89 -9
View File
@@ -3,11 +3,15 @@ package com.weather.modules.weather.dailydata.service.impl;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.weather.common.constant.Constant;
import com.weather.common.page.PageData;
import com.weather.common.redis.RedisKeys;
import com.weather.common.redis.RedisUtils;
import com.weather.common.service.impl.CrudServiceImpl;
import com.weather.common.utils.ConvertUtils;
import com.weather.common.utils.TimeUtils;
@@ -23,14 +27,18 @@ import com.weather.modules.weather.dailydata.service.WeatherDailyDataService;
import com.weather.modules.weather.station.dao.WeatherStationDao;
import com.weather.modules.weather.station.entity.WeatherStationEntity;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDate;
import java.time.ZoneId;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Slf4j
@Service
@AllArgsConstructor
public class WeatherDailyDataServiceImpl extends CrudServiceImpl<WeatherDailyDataDao, WeatherDailyDataEntity, WeatherDailyDataDto> implements WeatherDailyDataService {
@@ -38,6 +46,7 @@ public class WeatherDailyDataServiceImpl extends CrudServiceImpl<WeatherDailyDat
private final SysDeptService sysDeptService;
private final WeatherStationDao weatherStationDao;
private final WeatherDailyDataDao weatherDailyDataDao;
private final RedisUtils redisUtils;
@Override
public PageData<WeatherDailyDataDto> page(Map<String, Object> params) {
@@ -61,7 +70,7 @@ public class WeatherDailyDataServiceImpl extends CrudServiceImpl<WeatherDailyDat
private List<Long> getPrioritizedStationIds() {
UserDetail user = SecurityUser.getUser();
if (user == null || user.getDeptId() == null) return Collections.emptyList();
if (ObjectUtil.isEmpty(user) || user.getDeptId() == null) return Collections.emptyList();
List<Long> deptIds = sysDeptService.getSubDeptIdList(user.getDeptId());
if (deptIds.isEmpty()) return Collections.emptyList();
List<WeatherStationEntity> stations = weatherStationDao.selectList(
@@ -70,8 +79,11 @@ public class WeatherDailyDataServiceImpl extends CrudServiceImpl<WeatherDailyDat
.in("dept_id", deptIds));
return stations.stream()
.map(s -> {
try { return Long.valueOf(s.getStationCode()); }
catch (NumberFormatException e) { return null; }
try {
return Long.valueOf(s.getStationCode());
} catch (NumberFormatException e) {
return null;
}
})
.filter(Objects::nonNull)
.collect(Collectors.toList());
@@ -110,7 +122,7 @@ public class WeatherDailyDataServiceImpl extends CrudServiceImpl<WeatherDailyDat
.map(WeatherDailyDataEntity::getStationId)
.filter(Objects::nonNull)
.distinct()
.collect(Collectors.toList());
.toList();
Map<Long, Long> stationDeptMap = new HashMap<>();
if (!stationIds.isEmpty()) {
List<String> stationCodeList = stationIds.stream().map(String::valueOf).collect(Collectors.toList());
@@ -134,7 +146,8 @@ public class WeatherDailyDataServiceImpl extends CrudServiceImpl<WeatherDailyDat
entity.setUpdateDate(now);
}
weatherDailyDataDao.insertBatchMultiRow(list);
int successCount = weatherDailyDataDao.insertBatchMultiRow(list);
log.info("成功插入{}条数据",successCount);
return true;
}
@@ -169,11 +182,78 @@ public class WeatherDailyDataServiceImpl extends CrudServiceImpl<WeatherDailyDat
endTime = TimeUtils.convertToLocalDateTime(queryDto.getEndtDate());
}
QueryWrapper<WeatherDailyDataEntity> wrapper = new QueryWrapper<>();
wrapper.in("station_id", stationIds)
.between("observe_date", startTime, endTime);
// 尝试从缓存获取（仅"历年同月同日"查询可命中缓存）
List<DailyWeatherSummarizeDto> cached = getCachedSummarize(stationIds, startTime, endTime);
if (cached != null) {
return cached;
}
List<WeatherDailyDataEntity> list = weatherDailyDataDao.selectSummarizeList(wrapper);
// 缓存未命中，查询数据库
return querySummarizeFromDb(stationIds, startTime, endTime);
}
/**
* 尝试从 Redis 缓存获取汇总数据
*
* @return 缓存命中时返回数据，未命中返回 null
*/
@SuppressWarnings("unchecked")
private List<DailyWeatherSummarizeDto> getCachedSummarize(
List<Long> stationIds, LocalDate start, LocalDate end) {
// 仅当月-日相同的跨年查询可命中缓存
if (start.getMonth() != end.getMonth() || start.getDayOfMonth() != end.getDayOfMonth()) {
return null;
}
String key = RedisKeys.getWeatherSummarizeKey(end.getMonthValue(), end.getDayOfMonth());
Object cached = redisUtils.get(key);
if (cached == null) {
return null;
}
Map<String, List<DailyWeatherSummarizeDto>> grouped;
try {
grouped = (Map<String, List<DailyWeatherSummarizeDto>>) cached;
} catch (ClassCastException e) {
return null;
}
// 验证所有请求站点都在缓存中，任一缺失则降级到 DB
for (Long sid : stationIds) {
if (!grouped.containsKey(String.valueOf(sid))) {
return null;
}
}
// 从缓存中按站点过滤 + 限定年份范围（Key 为 String 避免 JSON 反序列化时 Long/Integer 类型丢失）
return stationIds.stream()
.flatMap(sid -> {
List<DailyWeatherSummarizeDto> stationData = grouped.get(String.valueOf(sid));
return stationData != null ? stationData.stream() : Stream.empty();
})
.filter(d -> {
LocalDate obsDate = d.getObserveDate().toInstant()
.atZone(ZoneId.systemDefault())
.toLocalDate();
return !obsDate.isBefore(start) && !obsDate.isAfter(end);
})
.collect(Collectors.toList());
}
/**
* 直接查询数据库获取汇总数据
* <p>用 MONTH/DAY 定位具体日期 + YEAR BETWEEN 限定年份范围，避免日期 BETWEEN 全量问题</p>
*/
private List<DailyWeatherSummarizeDto> querySummarizeFromDb(
List<Long> stationIds, LocalDate startTime, LocalDate endTime) {
List<WeatherDailyDataEntity> list = weatherDailyDataDao.selectSummarizeByMonthDay(
stationIds,
startTime.getMonthValue(),
startTime.getDayOfMonth(),
startTime.getYear(),
endTime.getYear());
return BeanUtil.copyToList(list, DailyWeatherSummarizeDto.class);
}
system-admin/src/main/java/com/weather/modules/weather/dailydata/task/WeatherSummarizeCacheTask.java
+65
View File
@@ -0,0 +1,65 @@
package com.weather.modules.weather.dailydata.task;
import cn.hutool.core.bean.BeanUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.weather.common.redis.RedisKeys;
import com.weather.common.redis.RedisUtils;
import com.weather.modules.job.task.ITask;
import com.weather.modules.weather.dailydata.dao.WeatherDailyDataDao;
import com.weather.modules.weather.dailydata.dto.DailyWeatherSummarizeDto;
import com.weather.modules.weather.dailydata.entity.WeatherDailyDataEntity;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.time.LocalDate;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* 天气汇总缓存预热任务
* <p>
* 每日凌晨 1:00 执行，将历年当月-当日的气象数据
* 按站点分组写入 Redis，供 export-sum 接口查询。
*
* @author 2333 123
* @since 1.0.0 2026-06-24
*/
@Slf4j
@Component("weatherSummarizeCacheTask")
public class WeatherSummarizeCacheTask implements ITask {
@Resource
private WeatherDailyDataDao weatherDailyDataDao;
@Resource
private RedisUtils redisUtils;
@Override
public void run(String params) {
LocalDate today = LocalDate.now();
int month = today.getMonthValue();
int day = today.getDayOfMonth();
log.info("开始刷新天气汇总缓存，日期: {}-{}", month, day);
try {
LambdaQueryWrapper<WeatherDailyDataEntity> wrapper = new LambdaQueryWrapper<>();
wrapper.apply("MONTH(observe_date) = {0} AND DAY(observe_date) = {1}", month, day);
List<WeatherDailyDataEntity> list = weatherDailyDataDao.selectSummarizeList(wrapper);
Map<String, List<DailyWeatherSummarizeDto>> grouped = list.stream()
.map(e -> BeanUtil.copyProperties(e, DailyWeatherSummarizeDto.class))
.collect(Collectors.groupingBy(dto -> String.valueOf(dto.getStationId())));
String key = RedisKeys.getWeatherSummarizeKey(month, day);
redisUtils.set(key, grouped, RedisUtils.NOT_EXPIRE);
log.info("天气汇总缓存刷新完成，共 {} 条记录，{} 个站点", list.size(), grouped.size());
} catch (Exception e) {
log.error("天气汇总缓存刷新失败", e);
}
}
}
system-admin/src/main/java/com/weather/modules/weather/filescan/FileScanStartupRunner.java
+21 -4
View File
@@ -7,7 +7,9 @@ import com.weather.modules.sys.entity.SysDeptEntity;
import com.weather.modules.sys.service.SysParamsService;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
import java.io.IOException;
@@ -15,18 +17,33 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.concurrent.TimeUnit;
@Slf4j
@Component
@AllArgsConstructor
public class FileScanStartupRunner implements CommandLineRunner {
public class FileScanStartupRunner {
/**
* 启动后延迟扫描时间（秒），给应用留出充分的初始化时间
*/
private static final long STARTUP_DELAY_SECONDS = 30;
private final SysParamsService sysParamsService;
private final SysDeptDao sysDeptDao;
private final FileWatchServiceManager fileWatchServiceManager;
@Override
public void run(String... args) {
@Async
@EventListener(ApplicationReadyEvent.class)
public void onApplicationReady() {
log.info("应用已启动，{} 秒后开始文件扫描目录初始化...", STARTUP_DELAY_SECONDS);
try {
TimeUnit.SECONDS.sleep(STARTUP_DELAY_SECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.warn("文件扫描延迟等待被中断，直接开始初始化");
}
String rootPath = sysParamsService.getValue(Constant.FILE_SCAN_ROOT_PATH);
if (StrUtil.isBlank(rootPath)) {
log.warn("FILE_SCAN_ROOT_PATH 未配置，跳过文件扫描目录初始化");
system-admin/src/main/resources/application-dev.yml
-20
View File
@@ -6,26 +6,6 @@ spring:
url: jdbc:mysql://localhost:3306/weather_data_system?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&nullCatalogMeansCurrent=true&rewriteBatchedStatements=true
username: root
password: root
#达梦8
# driver-class-name: dm.jdbc.driver.DmDriver
# url: jdbc:dm://192.168.10.10:5236/renren_security?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&nullCatalogMeansCurrent=true
# username: renren_security
# password: 12345678
# #Oracle
# driver-class-name: oracle.jdbc.OracleDriver
# url: jdbc:oracle:thin:@192.168.10.10:1521:xe
# username: renren_security
# password: 123456
# #SQLServer
# driver-class-name: com.microsoft.sqlserver.jdbc.SQLServerDriver
# url: jdbc:sqlserver://localhost:1433;DatabaseName=renren_security
# username: sa
# password: 123456
# #postgresql
# driver-class-name: org.postgresql.Driver
# url: jdbc:postgresql://192.168.10.10:5432/postgres
# username: postgres
# password: 123456
initial-size: 10
max-active: 100
min-idle: 10
system-admin/src/main/resources/application.yml
+2 -2
View File
@@ -40,7 +40,7 @@ spring:
data:
redis:
database: 0
host: 192.168.10.10
host: 127.0.0.1
port: 6379
password: # 密码（默认为空）
timeout: 6000ms # 连接超时时长（毫秒）
@@ -54,7 +54,7 @@ spring:
# 是否开启redis缓存 true开启 false关闭
project-options:
redis:
open: false
open: true
#mybatis
mybatis-plus:
system-admin/src/main/resources/mapper/dailyweather/WeatherDailyDataDao.xml
+13
View File
@@ -42,6 +42,19 @@
ORDER BY station_id, observe_date
</select>
<select id="selectSummarizeByMonthDay" resultMap="weatherDailyDataMap">
SELECT <include refid="summarizeColumns" />
FROM weather_daily_data
WHERE station_id IN
<foreach collection="stationIds" item="sid" open="(" separator="," close=")">
#{sid}
</foreach>
AND MONTH(observe_date) = #{month}
AND DAY(observe_date) = #{day}
AND YEAR(observe_date) BETWEEN #{startYear} AND #{endYear}
ORDER BY station_id, observe_date
</select>
<insert id="insertBatchMultiRow" parameterType="list" useGeneratedKeys="false">
INSERT INTO weather_daily_data (
id, station_id, observe_date,