update
This commit is contained in:
kzkzzzz
19
mygrpc/error.go
Normal file
19
mygrpc/error.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package mygrpc
|
||||
|
||||
import (
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
const (
|
||||
ServerTimeoutCode codes.Code = 10000
|
||||
ClientTimeoutCode codes.Code = 10001
|
||||
)
|
||||
|
||||
func GrpcClientTimeout(format string, args ...interface{}) error {
|
||||
return status.Newf(ClientTimeoutCode, format, args).Err()
|
||||
}
|
||||
|
||||
func GrpcServerTimeout(format string, args ...interface{}) error {
|
||||
return status.Newf(ServerTimeoutCode, format, args...).Err()
|
||||
}
|
||||
41
mygrpc/grpc.go
Normal file
41
mygrpc/grpc.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package mygrpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"google.golang.org/grpc/metadata"
|
||||
"google.golang.org/grpc/peer"
|
||||
"net"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultReadBufferSize = 256 * 1024 // 256kb
|
||||
DefaultWriteBufferSize = 256 * 1024
|
||||
|
||||
// DefaultWindowSize 滑动窗口 16mb 手动设置滑动窗口大小, 尝试提升吞吐量, 减少动态计算可能导致的cpu波动
|
||||
DefaultWindowSize = 16 * 1024 * 1024
|
||||
)
|
||||
|
||||
const (
|
||||
HeaderClientIP = "grpc-client-ip"
|
||||
HeaderServiceName = "grpc-service-name"
|
||||
)
|
||||
|
||||
func ClientIP(ctx context.Context) string {
|
||||
// 先从自定义的header获取
|
||||
md, ok := metadata.FromIncomingContext(ctx)
|
||||
if ok {
|
||||
if v := md.Get(HeaderClientIP); len(v) > 0 {
|
||||
return v[0]
|
||||
}
|
||||
}
|
||||
|
||||
p, ok := peer.FromContext(ctx)
|
||||
if ok {
|
||||
switch v := p.Addr.(type) {
|
||||
case *net.TCPAddr:
|
||||
return v.IP.String()
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
154
mygrpc/grpcc/client.go
Normal file
154
mygrpc/grpcc/client.go
Normal file
@@ -0,0 +1,154 @@
|
||||
package grpcc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"git.makemake.in/kzkzzzz/mycommon/myconf"
|
||||
"git.makemake.in/kzkzzzz/mycommon/mygrpc"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/keepalive"
|
||||
"google.golang.org/grpc/metadata"
|
||||
"google.golang.org/grpc/status"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ClientConf struct {
|
||||
useDefaultBufferCfg bool
|
||||
conf *myconf.Config
|
||||
grpcOpts []grpc.DialOption
|
||||
unaryMiddlewares []grpc.UnaryClientInterceptor
|
||||
}
|
||||
|
||||
type Opt func(*ClientConf)
|
||||
|
||||
func UseDefaultBufferCfg(v bool) Opt {
|
||||
return func(c *ClientConf) {
|
||||
c.useDefaultBufferCfg = v
|
||||
}
|
||||
}
|
||||
|
||||
func WithConf(v *myconf.Config) Opt {
|
||||
return func(c *ClientConf) {
|
||||
c.conf = v
|
||||
}
|
||||
}
|
||||
|
||||
func WithGrpcOpts(v ...grpc.DialOption) Opt {
|
||||
return func(c *ClientConf) {
|
||||
c.grpcOpts = v
|
||||
}
|
||||
}
|
||||
|
||||
func WithUnaryMiddlewares(v ...grpc.UnaryClientInterceptor) Opt {
|
||||
return func(c *ClientConf) {
|
||||
c.unaryMiddlewares = append(c.unaryMiddlewares, v...)
|
||||
}
|
||||
}
|
||||
|
||||
func MustNew(grpcUrl string, opts ...Opt) *grpc.ClientConn {
|
||||
client, err := New(grpcUrl, opts...)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
func New(grpcUrl string, opts ...Opt) (*grpc.ClientConn, error) {
|
||||
log.Printf("new grpc client url: %s", grpcUrl)
|
||||
|
||||
c := &ClientConf{
|
||||
useDefaultBufferCfg: true,
|
||||
unaryMiddlewares: []grpc.UnaryClientInterceptor{WrapRequestError()}, // 默认加上错误包装
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(c)
|
||||
}
|
||||
|
||||
dialOpts := []grpc.DialOption{
|
||||
grpc.WithKeepaliveParams(keepalive.ClientParameters{
|
||||
Time: time.Second * 20, // 如果没有activity,则每隔N s发送一个ping包
|
||||
Timeout: time.Second * 5, // 如果ping ack N s之内未返回则认为连接已断开
|
||||
PermitWithoutStream: true, // 如果没有active的stream,是否允许发送ping
|
||||
}),
|
||||
// 参考 https://github.com/grpc/grpc-go/tree/master/examples/features/load_balancing 设置轮训策略
|
||||
grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), // This sets the initial balancing policy.
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
}
|
||||
|
||||
// 使用默认的buffer调整配置
|
||||
if c.useDefaultBufferCfg {
|
||||
dialOpts = append(dialOpts,
|
||||
grpc.WithInitialWindowSize(mygrpc.DefaultWindowSize),
|
||||
grpc.WithInitialConnWindowSize(mygrpc.DefaultWindowSize),
|
||||
|
||||
grpc.WithReadBufferSize(mygrpc.DefaultReadBufferSize),
|
||||
grpc.WithWriteBufferSize(mygrpc.DefaultWriteBufferSize),
|
||||
|
||||
grpc.WithUnaryInterceptor(WrapRequestError()),
|
||||
)
|
||||
}
|
||||
|
||||
if len(c.unaryMiddlewares) > 0 {
|
||||
grpc.WithChainUnaryInterceptor(c.unaryMiddlewares...)
|
||||
}
|
||||
|
||||
if len(c.grpcOpts) > 0 {
|
||||
dialOpts = append(dialOpts, c.grpcOpts...)
|
||||
}
|
||||
|
||||
conn, err := grpc.NewClient(
|
||||
grpcUrl,
|
||||
dialOpts...,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
func WrapRequestError() grpc.UnaryClientInterceptor {
|
||||
return func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
|
||||
// 提取服务名称
|
||||
var serviceName string
|
||||
if md, ok := metadata.FromOutgoingContext(ctx); ok {
|
||||
if v := md.Get(mygrpc.HeaderServiceName); len(v) > 0 {
|
||||
serviceName = v[0]
|
||||
}
|
||||
}
|
||||
|
||||
err := invoker(ctx, method, req, reply, cc, opts...)
|
||||
if err != nil {
|
||||
|
||||
if serviceName != "" {
|
||||
return fmt.Errorf("request grpc err: [%s - %s] %s", serviceName, method, err)
|
||||
}
|
||||
|
||||
return fmt.Errorf("request grpc err: [%s] %s", method, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Timeout 客户端超时
|
||||
func Timeout(timeout time.Duration) grpc.UnaryClientInterceptor {
|
||||
return func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
|
||||
tCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
err := invoker(tCtx, method, req, reply, cc, opts...)
|
||||
|
||||
if v, ok := status.FromError(err); ok && v.Code() == codes.DeadlineExceeded {
|
||||
//return status.Errorf(grpcserver.Timeout, "call %s timeout %s", method, timeout)
|
||||
return mygrpc.GrpcClientTimeout("request timeout: %s", timeout)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
}
|
||||
331
mygrpc/grpcsr/server.go
Normal file
331
mygrpc/grpcsr/server.go
Normal file
@@ -0,0 +1,331 @@
|
||||
package grpcsr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"git.makemake.in/kzkzzzz/mycommon"
|
||||
"git.makemake.in/kzkzzzz/mycommon/graceful"
|
||||
"git.makemake.in/kzkzzzz/mycommon/myconf"
|
||||
"git.makemake.in/kzkzzzz/mycommon/mygrpc"
|
||||
"git.makemake.in/kzkzzzz/mycommon/mylog"
|
||||
"git.makemake.in/kzkzzzz/mycommon/myregistry"
|
||||
"github.com/spf13/pflag"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/health"
|
||||
"google.golang.org/grpc/health/grpc_health_v1"
|
||||
"google.golang.org/grpc/keepalive"
|
||||
"google.golang.org/grpc/reflection"
|
||||
"google.golang.org/grpc/status"
|
||||
"log"
|
||||
"net"
|
||||
|
||||
"runtime/debug"
|
||||
"time"
|
||||
)
|
||||
|
||||
//const DefaultInstanceName = "grpc"
|
||||
|
||||
var _ graceful.IRunner = (*Server)(nil)
|
||||
|
||||
type Conf struct {
|
||||
Addr string
|
||||
Port int
|
||||
Ip string
|
||||
Log bool
|
||||
}
|
||||
|
||||
type Opt func(server *Server)
|
||||
|
||||
type Server struct {
|
||||
gs *grpc.Server
|
||||
serviceId string
|
||||
serviceName string
|
||||
serverConf *Conf
|
||||
reg myregistry.IRegister
|
||||
grpcOpts []grpc.ServerOption
|
||||
logger mylog.ILogger
|
||||
registerGrpcFn func(*grpc.Server) // 注册grpc服务, 使用函数延迟调用, 便于先初始化中间件等操作
|
||||
|
||||
unaryMiddlewares []grpc.UnaryServerInterceptor // grpc一元服务端中间件
|
||||
|
||||
useDefaultBufferCfg bool
|
||||
delayStopMs int
|
||||
|
||||
serviceRegInfo *myregistry.ServiceInfo
|
||||
}
|
||||
|
||||
func UseDefaultBufferCfg(v bool) Opt {
|
||||
return func(server *Server) {
|
||||
server.useDefaultBufferCfg = v
|
||||
}
|
||||
}
|
||||
|
||||
func WithRegistry(serviceName string, reg myregistry.IRegister) Opt {
|
||||
return func(server *Server) {
|
||||
server.serviceName = serviceName
|
||||
server.reg = reg
|
||||
}
|
||||
}
|
||||
|
||||
func WithGrpcOpts(v ...grpc.ServerOption) Opt {
|
||||
return func(server *Server) {
|
||||
server.grpcOpts = v
|
||||
}
|
||||
}
|
||||
|
||||
func WithDelayStopMs(v int) Opt {
|
||||
return func(server *Server) {
|
||||
server.delayStopMs = v
|
||||
}
|
||||
}
|
||||
|
||||
func SetFlag() {
|
||||
pflag.Int("grpc.port", 18082, "listen port")
|
||||
pflag.String("grpc.log", "true", "enable request log")
|
||||
}
|
||||
|
||||
func New(cfg *myconf.Config, opts ...Opt) *Server {
|
||||
cf := &Conf{}
|
||||
err := cfg.UnmarshalKey("grpc", cf)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// 命令行的参数覆盖一次, Unmarshal解析的时候, 不会用命令行的参数覆盖 https://github.com/spf13/viper/issues/190
|
||||
cf.Port = cfg.GetInt(fmt.Sprintf("grpc.port"))
|
||||
cf.Log = cfg.GetBool(fmt.Sprintf("grpc.log"))
|
||||
|
||||
return NewByConf(cf, opts...)
|
||||
}
|
||||
|
||||
func NewByConf(conf *Conf, opts ...Opt) *Server {
|
||||
s := &Server{
|
||||
serverConf: conf,
|
||||
useDefaultBufferCfg: true,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(s)
|
||||
}
|
||||
|
||||
if s.logger == nil {
|
||||
s.logger = mylog.GetLogger()
|
||||
}
|
||||
|
||||
if s.reg != nil && s.serviceName == "" {
|
||||
panic("service name is empty")
|
||||
}
|
||||
|
||||
s.unaryMiddlewares = []grpc.UnaryServerInterceptor{
|
||||
s.grpcRecover(), // 默认启用recover中间件
|
||||
}
|
||||
|
||||
if s.serverConf.Log {
|
||||
s.unaryMiddlewares = append(s.unaryMiddlewares, s.requestLog())
|
||||
}
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *Server) Use(middlewares ...grpc.UnaryServerInterceptor) {
|
||||
s.unaryMiddlewares = append(s.unaryMiddlewares, middlewares...)
|
||||
}
|
||||
|
||||
func (s *Server) RegisterGrpc(fn func(*grpc.Server)) {
|
||||
s.registerGrpcFn = fn
|
||||
}
|
||||
|
||||
func (s *Server) initServer() {
|
||||
grpcOpts := []grpc.ServerOption{
|
||||
grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
|
||||
MinTime: time.Second * 5, // 如果客户端两次 ping 的间隔小于 N，则关闭连接
|
||||
PermitWithoutStream: true, // 即使没有 active stream, 也允许 ping
|
||||
}),
|
||||
grpc.KeepaliveParams(keepalive.ServerParameters{
|
||||
MaxConnectionIdle: time.Hour * 2, // 空闲连接时间
|
||||
MaxConnectionAgeGrace: time.Second * 30, // 在强制关闭连接之间, 允许有 N 的时间完成 pending 的 rpc 请求
|
||||
Time: time.Second * 20, // 如果一个连接空闲超过 N, 则发送一个 ping 请求
|
||||
Timeout: time.Second * 5, // 如果 ping 请求 N 内未收到回复, 则认为该连接已断开
|
||||
}),
|
||||
}
|
||||
|
||||
if s.useDefaultBufferCfg {
|
||||
grpcOpts = append(grpcOpts,
|
||||
grpc.InitialWindowSize(mygrpc.DefaultWindowSize),
|
||||
grpc.InitialConnWindowSize(mygrpc.DefaultWindowSize),
|
||||
|
||||
grpc.ReadBufferSize(mygrpc.DefaultReadBufferSize),
|
||||
grpc.WriteBufferSize(mygrpc.DefaultWriteBufferSize),
|
||||
)
|
||||
}
|
||||
|
||||
if len(s.unaryMiddlewares) > 0 {
|
||||
grpcOpts = append(grpcOpts, grpc.ChainUnaryInterceptor(s.unaryMiddlewares...))
|
||||
}
|
||||
|
||||
if len(s.grpcOpts) > 0 {
|
||||
grpcOpts = append(grpcOpts, s.grpcOpts...)
|
||||
}
|
||||
|
||||
s.gs = grpc.NewServer(grpcOpts...)
|
||||
|
||||
// 注册grpc服务
|
||||
s.registerGrpcFn(s.gs)
|
||||
}
|
||||
|
||||
func (s *Server) Run(ctx context.Context) error {
|
||||
s.initServer()
|
||||
|
||||
// 端口如果=0, 监听随机端口
|
||||
addr0 := fmt.Sprintf("%s:%d", s.serverConf.Addr, s.serverConf.Port)
|
||||
lis, err := net.Listen("tcp", addr0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 获取监听的端口
|
||||
port := lis.Addr().(*net.TCPAddr).Port
|
||||
|
||||
// 健康服务
|
||||
healthServer := health.NewServer()
|
||||
grpc_health_v1.RegisterHealthServer(s.gs, healthServer)
|
||||
|
||||
// 服务反射, 方便调试
|
||||
reflection.Register(s.gs)
|
||||
|
||||
var svcIp = s.serverConf.Ip
|
||||
if svcIp == "" {
|
||||
svcIp = mycommon.GetOutboundIP()
|
||||
}
|
||||
|
||||
// 注册服务
|
||||
if s.reg != nil {
|
||||
s.serviceRegInfo = &myregistry.ServiceInfo{
|
||||
ServiceName: s.serviceName,
|
||||
Ip: svcIp,
|
||||
Port: port,
|
||||
}
|
||||
|
||||
err = s.reg.Register(s.serviceRegInfo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
addr := fmt.Sprintf("%s:%d", s.serverConf.Addr, port)
|
||||
log.Printf("grpc server listen on %s", addr)
|
||||
|
||||
err = s.gs.Serve(lis)
|
||||
if err != nil {
|
||||
log.Printf("start grpc server err: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) Stop() {
|
||||
if s.reg != nil {
|
||||
err := s.reg.Deregister(s.serviceRegInfo)
|
||||
if err != nil {
|
||||
s.logger.Errorf("grpc server deregister err: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 如果使用k8s service, 关闭pod和往service注销ip是同时进行的, 如果退出服务比注销ip先完成, 可能有流量继续进来, 导致请求失败
|
||||
// 延迟一段时间, 确保服务已经注销ip, 再关闭服务
|
||||
|
||||
// 如何使用注册中心, 先从中心退出ip, 也延迟一段时间, 等上游网关更新ip完成(正常不会太久), 不会有流量进来旧服务, 再退出服务
|
||||
if s.delayStopMs > 0 {
|
||||
delayTime := time.Millisecond * time.Duration(s.delayStopMs)
|
||||
log.Printf("grpc server delay stop: %s", delayTime)
|
||||
time.Sleep(delayTime)
|
||||
}
|
||||
|
||||
s.gs.GracefulStop()
|
||||
|
||||
log.Printf("grpc server stop")
|
||||
}
|
||||
|
||||
type handleResp struct {
|
||||
resp interface{}
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *Server) requestLog() grpc.UnaryServerInterceptor {
|
||||
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
|
||||
start := time.Now()
|
||||
|
||||
resp, err := handler(ctx, req)
|
||||
|
||||
var (
|
||||
code codes.Code
|
||||
codeMsg = "OK"
|
||||
)
|
||||
if err != nil {
|
||||
fromError, ok := status.FromError(err)
|
||||
if ok {
|
||||
code = fromError.Code()
|
||||
} else {
|
||||
code = status.New(codes.Unknown, err.Error()).Code()
|
||||
}
|
||||
|
||||
codeMsg = fmt.Sprintf("Error Code: %s(%d)", code.String(), uint32(code))
|
||||
}
|
||||
|
||||
s.logger.Infof(
|
||||
"%s - %s - %s - %s",
|
||||
codeMsg, time.Since(start), mygrpc.ClientIP(ctx), info.FullMethod,
|
||||
)
|
||||
|
||||
return resp, err
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) grpcRecover() grpc.UnaryServerInterceptor {
|
||||
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
|
||||
defer func() {
|
||||
if err0 := recover(); err0 != nil {
|
||||
log.Printf("%s - panic: %v\n%s", info.FullMethod, err0, debug.Stack())
|
||||
err = fmt.Errorf("server err: %s - system err: %s", info.FullMethod, err0)
|
||||
}
|
||||
}()
|
||||
|
||||
return handler(ctx, req)
|
||||
}
|
||||
}
|
||||
|
||||
// Timeout 控制服务端超时
|
||||
func Timeout(timeout time.Duration) grpc.UnaryServerInterceptor {
|
||||
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
|
||||
tCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
ch := make(chan *handleResp, 1)
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
if err0 := recover(); err0 != nil {
|
||||
log.Printf("%s - panic: %v\n%s", info.FullMethod, err0, debug.Stack())
|
||||
ch <- &handleResp{nil, fmt.Errorf("server err: %s - system err: %s", info.FullMethod, err0)}
|
||||
}
|
||||
}()
|
||||
//start := time.Now()
|
||||
r := &handleResp{}
|
||||
r.resp, r.err = handler(tCtx, req)
|
||||
//log.Printf("rta server time: %s", time.Since(start))
|
||||
ch <- r
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-tCtx.Done():
|
||||
return nil, mygrpc.GrpcServerTimeout("server err: grpc handle timeout %s %s", timeout, info.FullMethod)
|
||||
//return nil, status.Errorf(codes.DeadlineExceeded, "grpc handle timeout %s %s", timeout, info.FullMethod)
|
||||
|
||||
case res := <-ch:
|
||||
return res.resp, res.err
|
||||
}
|
||||
|
||||
//return nil, fmt.Errorf("handle err %s.%s", info.Server, info.FullMethod)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user