环境：
spring-boot-starter-* 1.5.2.RELEASE
spring-cloud-starter-eureka-* 1.2.6.RELEASE.jar
spring-cloud-starter-hystrix-* 1.2.6.RELEASE
spring-cloud-starter-feign 1.2.6.RELEASE
spring-cloud-starter-ribbon 1.2.6.RELEASE
spring-cloud-starter-zuul 1.2.6.RELEASE
访问链路: nginx->zuul网关->微服务A
现象：jmeter 600个线程压测请求爆红，错误率45%以上，查看对应请求的zuul网关，部分日志如下：
Caused by: java.lang.RuntimeException: could not acquire a semaphore for execution
at com.netflix.hystrix.AbstractCommand.handleSemaphoreRejectionViaFallback(AbstractCommand.java:949) ~[hystrix-core-1.5.6.
at com.netflix.hystrix.AbstractCommand.applyHystrixSemantics(AbstractCommand.java:540) ~[hystrix-core-1.5.6.jar!/:1.5.6]
at com.netflix.hystrix.AbstractCommand.access$100(AbstractCommand.java:59) ~[hystrix-core-1.5.6.jar!/:1.5.6]
at com.netflix.hystrix.AbstractCommand$4.call(AbstractCommand.java:405) ~[hystrix-core-1.5.6.jar!/:1.5.6]
at com.netflix.hystrix.AbstractCommand$4.call(AbstractCommand.java:402) ~[hystrix-core-1.5.6.jar!/:1.5.6]
at rx.internal.operators.OnSubscribeDefer.call(OnSubscribeDefer.java:46) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.internal.operators.OnSubscribeDefer.call(OnSubscribeDefer.java:35) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.Observable.unsafeSubscribe(Observable.java:10211) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:48) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:33) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.Observable.unsafeSubscribe(Observable.java:10211) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.internal.operators.OnSubscribeDoOnEach.call(OnSubscribeDoOnEach.java:41) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.internal.operators.OnSubscribeDoOnEach.call(OnSubscribeDoOnEach.java:30) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:48) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.Observable.unsafeSubscribe(Observable.java:10211) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.internal.operators.OnSubscribeDoOnEach.call(OnSubscribeDoOnEach.java:41) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.internal.operators.OnSubscribeDoOnEach.call(OnSubscribeDoOnEach.java:30) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.Observable.unsafeSubscribe(Observable.java:10211) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.internal.operators.OnSubscribeDefer.call(OnSubscribeDefer.java:51) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.internal.operators.OnSubscribeDefer.call(OnSubscribeDefer.java:35) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:48) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.Observable.subscribe(Observable.java:10307) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.Observable.subscribe(Observable.java:10274) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.internal.operators.BlockingOperatorToFuture.toFuture(BlockingOperatorToFuture.java:51) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.observables.BlockingObservable.toFuture(BlockingObservable.java:412) ~[rxjava-1.1.10.jar!/:1.1.10]
at com.netflix.hystrix.HystrixCommand.queue(HystrixCommand.java:377) ~[hystrix-core-1.5.6.jar!/:1.5.6]
at com.netflix.hystrix.HystrixCommand.execute(HystrixCommand.java:343) ~[hystrix-core-1.5.6.jar!/:1.5.6]
at org.springframework.cloud.netflix.zuul.filters.route.RibbonRoutingFilter.forward(RibbonRoutingFilter.java:139) ~[spring
... 67 more
2020-09-01 15:54:07,361:WARN org.springframework.cloud.netflix.zuul.filters.post.SendErrorFilter http-nio-6100-exec-70 (SendErrorF
com.netflix.zuul.exception.ZuulException: Forwarding error
at org.springframework.cloud.netflix.zuul.filters.route.RibbonRoutingFilter.handleException(RibbonRoutingFilter.java:170)
at org.springframework.cloud.netflix.zuul.filters.route.RibbonRoutingFilter.forward(RibbonRoutingFilter.java:145) ~[spring
at org.springframework.cloud.netflix.zuul.filters.route.RibbonRoutingFilter.run(RibbonRoutingFilter.java:88) ~[spring-clou
at com.netflix.zuul.ZuulFilter.runFilter(ZuulFilter.java:112) [zuul-core-1.3.0.jar!/:1.3.0]
at com.netflix.zuul.FilterProcessor.processZuulFilter(FilterProcessor.java:193) [zuul-core-1.3.0.jar!/:1.3.0]
at com.netflix.zuul.FilterProcessor.runFilters(FilterProcessor.java:157) [zuul-core-1.3.0.jar!/:1.3.0]
at com.netflix.zuul.FilterProcessor.route(FilterProcessor.java:118) [zuul-core-1.3.0.jar!/:1.3.0]
at com.netflix.zuul.ZuulRunner.route(ZuulRunner.java:96) [zuul-core-1.3.0.jar!/:1.3.0]
at com.netflix.zuul.http.ZuulServlet.route(ZuulServlet.java:116) [zuul-core-1.3.0.jar!/:1.3.0]
at com.netflix.zuul.http.ZuulServlet.service(ZuulServlet.java:81) [zuul-core-1.3.0.jar!/:1.3.0]
at org.springframework.web.servlet.mvc.ServletWrappingController.handleRequestInternal(ServletWrappingController.java:157)
at org.springframework.cloud.netflix.zuul.web.ZuulController.handleRequest(ZuulController.java:44) [spring-cloud-netflix-c
at org.springframework.web.servlet.mvc.SimpleControllerHandlerAdapter.handle(SimpleControllerHandlerAdapter.java:50) [spri
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:963) [spring-webmvc-4.3.7.RELEASE.j
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897) [spring-webmvc-4.3.7.RELEASE.ja
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970) [spring-webmvc-4.3.7.RELEASE
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:861) [spring-webmvc-4.3.7.RELEASE.jar!/:4.
at javax.servlet.http.HttpServlet.service(HttpServlet.java:622) [tomcat-embed-core-8.5.11.jar!/:8.5.11]
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846) [spring-webmvc-4.3.7.RELEASE.jar!/:
at javax.servlet.http.HttpServlet.service(HttpServlet.java:729) [tomcat-embed-core-8.5.11.jar!/:8.5.11]
查看hystrix配置：
hystrix:
command:
default:
execution:
isolation:
thread:
timeoutInMilliseconds: 10000
从hystrix-core包查看
com.netflix.hystrix.config.HystrixThreadPoolConfiguration
com.netflix.hystrix.HystrixThreadPoolProperties
可以知道
coreSize、maxQueueSize、queueSizeRejectionThreshold
的配置方式
hystrix.threadpool.default.coreSize=100 // 线程池大小，默认10
hystrix.threadpool.default.maxQueueSize=2000 // 请求队列大小，默认-1 ，表示不开启队列（该数值无法动态更改，因此使用'queueSizeRejectionThreshold'人为地限制和拒绝）
hystrix.threadpool.default.queueSizeRejectionThreshold=2000 // 队列中，默认5，
关键代码如下
public abstract class HystrixThreadPoolProperties {
/* defaults */
private Integer default_coreSize = 10; // size of thread pool
...
private Integer default_maxQueueSize = -1; // size of queue (this can't be dynamically changed so we use 'queueSizeRejectionThreshold' to artificially limit and reject)
// -1 turns if off and makes us use SynchronousQueue
private Integer default_queueSizeRejectionThreshold = 5; // number of items in queue
...
protected HystrixThreadPoolProperties(HystrixThreadPoolKey key, Setter builder, String propertyPrefix) {
this.corePoolSize = getProperty(propertyPrefix, key, "coreSize", builder.getCoreSize(), default_coreSize);
...
this.maxQueueSize = getProperty(propertyPrefix, key, "maxQueueSize", builder.getMaxQueueSize(), default_maxQueueSize);
this.queueSizeRejectionThreshold = getProperty(propertyPrefix, key, "queueSizeRejectionThreshold", builder.getQueueSizeRejectionThreshold(), default_queueSizeRejectionThreshold);
...
}
private static HystrixProperty<Integer> getProperty(String propertyPrefix, HystrixThreadPoolKey key, String instanceProperty, Integer builderOverrideValue, Integer defaultValue) {
return forInteger()
.add(propertyPrefix + ".threadpool." + key.name() + "." + instanceProperty, builderOverrideValue)
.add(propertyPrefix + ".threadpool.default." + instanceProperty, defaultValue)
.build();
}
}
...
一顿操作猛如虎，然而令人郁闷的是单独配置hystrix然并卵，最终通过zuul.semaphore.max-semaphores配置信号量，压测2000线程，请求错误率0.2%（微服务A访问超时）
hystrix:
command:
default:
execution:
isolation:
thread:
timeoutInMilliseconds: 10000
threadpool:
default:
coreSize: 100
maxQueueSize: 2000
queueSizeRejectionThreshold: 2000
zuul:
semaphore:
max-semaphores: 5000
如果出现以下报错，则需要注意ribbon的超时配置
Caused by: java.util.concurrent.TimeoutException
at com.netflix.hystrix.AbstractCommand.handleTimeoutViaFallback(AbstractCommand.java:980) ~[hystrix-core-1.5.6.jar!/:1.5.6]
at com.netflix.hystrix.AbstractCommand.access$500(AbstractCommand.java:59) ~[hystrix-core-1.5.6.jar!/:1.5.6]
at com.netflix.hystrix.AbstractCommand$12.call(AbstractCommand.java:595) ~[hystrix-core-1.5.6.jar!/:1.5.6]
at com.netflix.hystrix.AbstractCommand$12.call(AbstractCommand.java:587) ~[hystrix-core-1.5.6.jar!/:1.5.6]
at rx.internal.operators.OperatorOnErrorResumeNextViaFunction$4.onError(OperatorOnErrorResumeNextViaFunction.java:140) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.internal.operators.OnSubscribeDoOnEach$DoOnEachSubscriber.onError(OnSubscribeDoOnEach.java:87) ~[rxjava-1.1.10.jar!/:1.1.10]
at rx.internal.operators.OnSubscribeDoOnEach$DoOnEachSubscriber.onError(OnSubscribeDoOnEach.java:87) ~[rxjava-1.1.10.jar!/:1.1.10]
at com.netflix.hystrix.AbstractCommand$HystrixObservableTimeoutOperator$1.run(AbstractCommand.java:1121) ~[hystrix-core-1.5.6.jar!/:1.5.6]
at com.netflix.hystrix.strategy.concurrency.HystrixContextRunnable$1.call(HystrixContextRunnable.java:41) ~[hystrix-core-1.5.6.jar!/:1.5.6]
at com.netflix.hystrix.strategy.concurrency.HystrixContextRunnable$1.call(HystrixContextRunnable.java:37) ~[hystrix-core-1.5.6.jar!/:1.5.6]
at com.netflix.hystrix.strategy.concurrency.HystrixContextRunnable.run(HystrixContextRunnable.java:57) ~[hystrix-core-1.5.6.jar!/:1.5.6]
at com.netflix.hystrix.AbstractCommand$HystrixObservableTimeoutOperator$2.tick(AbstractCommand.java:1138) ~[hystrix-core-1.5.6.jar!/:1.5.6]
at com.netflix.hystrix.util.HystrixTimer$1.run(HystrixTimer.java:99) ~[hystrix-core-1.5.6.jar!/:1.5.6]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_241]
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) ~[?:1.8.0_241]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_241]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) ~[?:1.8.0_241]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_241]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_241]
原来
ribbon:
ConnectTimeout: 3000
ReadTimeout: 60000
改为
ribbon:
ConnectTimeout: 10000
ReadTimeout: 60000
MaxConnectionsPerHost: 3000
MaxTotalConnections: 3000
参考
https://www.cnblogs.com/java-spring/p/13042335.html
https://www.cnblogs.com/killerqi/p/10906392.html
https://blog.csdn.net/weixin_41231928/article/details/105238389
https://blog.csdn.net/wang20y8/article/details/102523265
