I had a bit of hassle with Guacamole too.

I eventually got it working like this:

location / {
    proxy_pass                             http://guacamole:8080/guacamole/;
    proxy_buffering                        off;
    proxy_http_version                     1.1;
    proxy_cookie_path /guacamole/          /;
    proxy_set_header  X-Forwarded-Host     $host;
    proxy_set_header  X-Forwarded-Server   $host;
    proxy_set_header  X-Forwarded-For      $proxy_add_x_forwarded_for;
    proxy_set_header  X-Forwarded-Proto    $scheme;
    proxy_set_header  Host                 $host;
    proxy_set_header  Upgrade              $http_upgrade;
    proxy_set_header  Connection           $http_connection;
}

Happy to send you my docker-compose.yml if you want a copy.