记录一次生产数据积压的bug

场景：
程序从kafka中拉取数据，topic分区数6，程序有两台，某段时间数据一直积压无法恢复。
问题原因：
程序开启了日志忘记关闭，导致大量的io，程序消费速度变慢。
方案：
程序日志修改为异步的方式，但是这样会导致日志丢失的情况，因为业务上没有需要日志完全保留，所以采取这种方式，下面是异步日志的配置：

  <configuration scan="false" scanPeriod="30 seconds"  debug="false">

    <property name="contextName" value="xxx"/>
    <contextName>${contextName}</contextName>
    <property name="LOG_HOME" value="log"/>

    <appender name="HOUR_ROLLING" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <File>${LOG_HOME}/${contextName}.log</File>
        <encoder>
            <pattern>%date [%thread] %-5level [%file:%line] %msg%n </pattern>
            <charset>UTF-8</charset>
        </encoder>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>${LOG_HOME}/${contextName}-%d{yyyy-MM-dd-HH}.%i.gz</fileNamePattern>
            <TimeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
                <maxFileSize>512MB</maxFileSize>
            </TimeBasedFileNamingAndTriggeringPolicy>
            <!--最多保留7天，即168小时的日志文件-->
            <maxHistory>168</maxHistory>
        </rollingPolicy>
    </appender>

    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <pattern>%date [%thread] %-5level [%file:%line] %msg%n</pattern></encoder>
    </appender>

    <appender name="ERROR" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <FileNamePattern>${LOG_HOME}/${contextName}-error.%d{yyyy-MM-dd}.log</FileNamePattern>
            <maxHistory>720</maxHistory>
        </rollingPolicy>
        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %msg%n</pattern>
            <charset class="java.nio.charset.Charset">UTF-8</charset>
        </encoder>
        <filter class="ch.qos.logback.classic.filter.LevelFilter"><!-- 只打印错误日志 -->
            <level>ERROR</level>
            <onMatch>ACCEPT</onMatch>
            <onMismatch>DENY</onMismatch>
        </filter>
    </appender>

    <appender name="LOG_ASYNC" class= "ch.qos.logback.classic.AsyncAppender">
        <discardingThreshold>0</discardingThreshold>
        <queueSize>4096</queueSize>
        <includeCallerData>true</includeCallerData>
        <appender-ref ref ="HOUR_ROLLING"/>
    </appender>

    <appender name="ERROR_ASYNC" class= "ch.qos.logback.classic.AsyncAppender">
        <discardingThreshold>0</discardingThreshold>
        <queueSize>4096</queueSize>
        <includeCallerData>true</includeCallerData>
        <appender-ref ref ="ERROR"/>
    </appender>


    <root level="INFO">
        <appender-ref ref="LOG_ASYNC" />
        <appender-ref ref="ERROR_ASYNC" />
        <appender-ref ref="STDOUT" />
    </root>
</configuration>

最后编辑于：2020.05.06 11:24:27