How to deploy multiple skygear-servers


#1

I have set up a master skygear-server,and two skygear-server in slave mode.
nginx as reverse proxy.
set http://skygear-server.com/pubsub and http://skygear-server.com/_/pubsub request go to master.
and other requests go to master and slave。

but send message is not stable,its always say"チャットが見つかりませんでした Error". No chat was found.but sometime it is normal.

If I set up multiple skygear-server+chat,how to change my configuration file,Does nginx still need improvement?

If I set up mutilple skygear-server+chat both in master mode. Is this possible?


#2

Your nginx setup looks good. For the chat plugin setting, please make sure each skygear-server (master and slave) have its own chat plugin connected. So if you have 1 master and 2 slave skygear-server, you should have 3 py-skygear, one for each. Are you setting up this way? Let us know for further questions:)


#3

yes, I have 3 py-skygear.

I found the nginx is incorrect. I have set /pubsub、/_/pubsub go to master,but it still go to slave when I
access /pubsub with browser. and it is also go to slave with Demo of Skygear Chat plugin with JS-SDK.
can you check my nginx ? thanks!!!

    location ~ /(pubsub|_/pubsub) {
        proxy_pass http://skygear-master;

        proxy_redirect     off;
        proxy_http_version 1.1;
        proxy_set_header   Upgrade          $http_upgrade;
        proxy_set_header   Connection       "upgrade";
        proxy_set_header   Host             $host;
        proxy_set_header   X-Real-IP        $remote_addr;
        proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
        proxy_max_temp_file_size 0;
        proxy_connect_timeout      90;
        proxy_send_timeout         90;
        proxy_read_timeout         90;
        proxy_buffer_size          4k;
        proxy_buffers              4 32k;
        proxy_busy_buffers_size    64k;
        proxy_temp_file_write_size 64k;
    }

    location / {
        proxy_pass http://skygear-all;

        proxy_redirect     off;
        proxy_http_version 1.1;
        proxy_set_header   Upgrade          $http_upgrade;
        proxy_set_header   Connection       "upgrade";
        proxy_set_header   Host             $host;
        proxy_set_header   X-Real-IP        $remote_addr;
        proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
        proxy_max_temp_file_size 0;
        proxy_connect_timeout      90;
        proxy_send_timeout         90;
        proxy_read_timeout         90;
        proxy_buffer_size          4k;
        proxy_buffers              4 32k;
        proxy_busy_buffers_size    64k;
        proxy_temp_file_write_size 64k;
    }

#4

@leeqium I tried copy your config and added upstream skygear-master and skygear-all, it works for me. Would you mind providing the full nginx config file and just remove the data that you think it is sensitive?


#5
upstream skygear-master {
    server 127.0.0.1:3000; #master
}
upstream skygear-all {
    server 127.0.0.1:3000; #master
    server 127.0.0.1:3001; #node1
    server 127.0.0.1:3002; #node2
}

server {
    listen       80 default_server;
    listen       [::]:80 default_server;
    server_name  example.com;
    root         /var/www/html;

    access_log /var/log/nginx/skygear.access.log main;
    error_log /var/log/nginx/skygear.error.log;

    # Load configuration files for the default server block.
    #include /etc/nginx/default.d/*.conf;

    # https
    listen       443 ssl http2;
    ssl_certificate "/usr/share/nginx/ssl/localhost.crt";
    ssl_certificate_key "/usr/share/nginx/ssl/localhost.key";
    ssl_session_cache shared:SSL:1m;
    ssl_session_timeout  10m;
    ssl_ciphers HIGH:!aNULL:!MD5;
    ssl_prefer_server_ciphers on;

    location ~ /(pubsub|_/pubsub) {
        proxy_pass http://skygear-master;

        proxy_redirect     off;
        proxy_http_version 1.1;
        proxy_set_header   Upgrade          $http_upgrade;
        proxy_set_header   Connection       "upgrade";
        proxy_set_header   Host             $host;
        proxy_set_header   X-Real-IP        $remote_addr;
        proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
        proxy_max_temp_file_size 0;
        proxy_connect_timeout      90;
        proxy_send_timeout         90;
        proxy_read_timeout         90;
        proxy_buffer_size          4k;
        proxy_buffers              4 32k;
        proxy_busy_buffers_size    64k;
        proxy_temp_file_write_size 64k;
    }
    location / {
        proxy_pass http://skygear-all;

        proxy_redirect     off;
        proxy_http_version 1.1;
        proxy_set_header   Upgrade          $http_upgrade;
        proxy_set_header   Connection       "upgrade";
        proxy_set_header   Host             $host;
        proxy_set_header   X-Real-IP        $remote_addr;
        proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
        proxy_max_temp_file_size 0;
        proxy_connect_timeout      90;
        proxy_send_timeout         90;
        proxy_read_timeout         90;
        proxy_buffer_size          4k;
        proxy_buffers              4 32k;
        proxy_busy_buffers_size    64k;
        proxy_temp_file_write_size 64k;
    }
}

above is full nginx config. thanks!


#6
time="2018-05-29T19:26:09+09:00" level=debug msg="GET /pubsub" logger=router
time="2018-05-29T19:26:09+09:00" level=debug msg="------ Header: ------" logger=router
time="2018-05-29T19:26:09+09:00" level=debug msg="Upgrade: [websocket]" logger=router
time="2018-05-29T19:26:09+09:00" level=debug msg="Connection: [Upgrade]" logger=router
time="2018-05-29T19:26:09+09:00" level=debug msg="Origin: [http://127.0.0.1:3001]" logger=router
time="2018-05-29T19:26:09+09:00" level=debug msg="Sec-Websocket-Key: [NvxN6EO3ksE8V7aFCan3Zw==]" logger=router
time="2018-05-29T19:26:09+09:00" level=debug msg="Sec-Websocket-Version: [13]" logger=router
time="2018-05-29T19:26:09+09:00" level=debug msg="X-Skygear-Api-Key: [api-key]" logger=router
time="2018-05-29T19:26:09+09:00" level=debug msg="------ Request: ------" logger=router
time="2018-05-29T19:26:09+09:00" level=debug msg="0 bytes of request body" logger=router
time="2018-05-29T19:26:09+09:00" level=debug msg="------ Response: ------" logger=router
time="2018-05-29T19:26:09+09:00" level=debug msg="{\"result\":{\"status\":\"OK\"}}\n" logger=router
time="2018-05-29T19:26:09+09:00" level=debug msg="------ Response: ------" logger=router
time="2018-05-29T19:26:09+09:00" level=debug msg="{\"error\":{\"name\":\"UnexpectedError\",\"code\":10000,\"message\":\"Handshake status 200 OK\",\"info\":{\"trace\":\"Tracebac
k (most recent call last):\\n  File \\\"/usr/lib/python3.6/site-packages/skygear/transmitter/common.py\\\", line 40, in wrapper\\n    return dict(result=f(self, *args, **kwargs
))\\n  File \\\"/usr/lib/python3.6/site-packages/skygear/transmitter/common.py\\\", line 100, in call_func\\n    return self.op(obj, param.get('args', {}))\\n  File \\\"/usr/li
b/python3.6/site-packages/skygear/transmitter/common.py\\\", line 174, in op\\n    return func(*args, **kwargs)\\n  File \\\"chat/receipt_handlers.py\\\", line 267, in mark_as_
read_lambda\\n    return handle_mark_as_read(arg)\\n  File \\\"chat/receipt_handlers.py\\\", line 41, in handle_mark_as_read\\n    mark_messages(message_ids, True, True)\\n  Fi
le \\\"chat/receipt_handlers.py\\\", line 99, in mark_messages\\n    __update_and_notify_messages(new_message_ids)\\n  File \\\"chat/receipt_handlers.py\\\", line 227, in __upd
ate_and_notify_messages\\n    message.notifyParticipants()\\n  File \\\"chat/message.py\\\", line 105, in notifyParticipants\\n    self)\\n  File \\\"chat/pubsub.py\\\", line 3
1, in _publish_record_event\\n    'record': serialize_record(record)\\n  File \\\"chat/pubsub.py\\\", line 17, in _publish_event\\n    'data': data\\n  File \\\"chat/hub.py\\\"
, line 61, in publish\\n    conn = create_connection(self.end_point, **wsopts)\\n  File \\\"/usr/lib/python3.6/site-packages/websocket/_core.py\\\", line 494, in create_connect
ion\\n    websock.connect(url, **options)\\n  File \\\"/usr/lib/python3.6/site-packages/websocket/_core.py\\\", line 220, in connect\\n    self.handshake_response = handshake(s
elf.sock, *addrs, **options)\\n  File \\\"/usr/lib/python3.6/site-packages/websocket/_handshake.py\\\", line 69, in handshake\\n    status, resp = _get_resp_headers(sock)\\n  F
ile \\\"/usr/lib/python3.6/site-packages/websocket/_handshake.py\\\", line 135, in _get_resp_headers\\n    raise WebSocketBadStatusException(\\\"Handshake status %d %s\\\", sta
tus, status_message)\\nwebsocket._exceptions.WebSocketBadStatusException: Handshake status 200 OK\\n\"}}}\n" logger=router

above skygear log


#7

I tested your nginx config and it looks good. All /pubsub and /_/pubsub go to the master.

For the error, I suspect the chat plugin connect the pubsub incorrectly. For the env variable of py-skygear, please set PUBSUB_URL to the nginx url. For example, ws://example.com:3000/pubsub. See if it works?


#8

Thank you very much! It works!
The PUBSUB_URL variable of master and slave are the same: ws://example.com:3000/pubsub

By the way, If i set up 3 master, how to deploy them. Is the PUBSUB_URL variable of py-skygear the same for 3 master? For the nginx config, if set ip-hash for upstream , the requests from the same client will always be directed to the same server. May it solve the problem of 3 master fights?


#9

@leeqium

Great to hear that it works for you!

I am sorry that skygear-server doesn’t support multiple masters yet, in the current architecture pubsub will only distribute events within the same node. And also the scheduled tasks setup also only support single master node. So setting ip-hash for upstream still cannot solve the problem.


#10

oh,what a pity. Thank you all the same.


#11

@leeqium A relevant feature issue was created: https://github.com/SkygearIO/features/issues/230