From 6e2ed858e457cd10d5276ba3749ee2faced54669 Mon Sep 17 00:00:00 2001 From: Erik Brakkee Date: Tue, 23 Jul 2024 20:47:51 +0200 Subject: [PATCH] fileserver now uses go template language. updated docs for windows. --- cmd/converge/fileserver.go | 98 ++++++++------- cmd/converge/server.go | 43 +------ static/index.html | 236 +++++++++++++++++++++---------------- 3 files changed, 190 insertions(+), 187 deletions(-) diff --git a/cmd/converge/fileserver.go b/cmd/converge/fileserver.go index 5009e53..1d38bec 100644 --- a/cmd/converge/fileserver.go +++ b/cmd/converge/fileserver.go @@ -1,59 +1,67 @@ package main import ( - "bytes" - "io" + "html/template" "net/http" + "os" + "path/filepath" "strings" ) -// filters to filter html content. -var filters map[string]string = make(map[string]string) - -type FilteredFileSystem struct { - fs http.FileSystem +type FileHandlerFilter struct { + dir string + fileHandler http.Handler } -func (ffs FilteredFileSystem) Open(name string) (http.File, error) { - f, err := ffs.fs.Open(name) +func NewFileHandler(dir string) *FileHandlerFilter { + handler := FileHandlerFilter{ + dir: dir, + fileHandler: http.FileServer(http.Dir(dir)), + } + return &handler +} + +func (handler FileHandlerFilter) ServeHTTP(w http.ResponseWriter, r *http.Request) { + contextPath := r.URL.Path + path := filepath.Join(handler.dir, contextPath) + fileInfo, err := os.Stat(path) + if err == nil && fileInfo.IsDir() { + contextPath = filepath.Join(contextPath, "index.html") + } + + if !strings.HasSuffix(contextPath, ".html") { + handler.fileHandler.ServeHTTP(w, r) + return + } + + w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate") + w.Header().Set("Pragma", "no-cache") + w.Header().Set("Expires", "0") + + filters := make(map[string]string) + if r.TLS == nil { + filters["secure"] = "" + } else { + filters["secure"] = "s" + } + for _, header := range []string{"X-Forwarded-Proto", "X-Scheme", "X-Forwarded-Scheme"} { + values := r.Header.Values(header) + for _, value := range values { + if strings.ToLower(value) == "https" { + filters["secure"] = "s" + } + } + } + filters["host"] = r.Host + + tmpl, err := template.ParseFiles(filepath.Join(handler.dir, contextPath)) if err != nil { - return nil, err + // let the filehandler generate the rror + handler.fileHandler.ServeHTTP(w, r) } - if !strings.HasSuffix(name, ".html") { - return f, nil - } - return NewFilteredFile(f, filters) -} - -type FilteredFile struct { - http.File - - // Read bytes 0..(index-1) already - index int - fullContents *bytes.Buffer -} - -func NewFilteredFile(f http.File, filter map[string]string) (FilteredFile, error) { - file := FilteredFile{ - index: 0, - fullContents: bytes.NewBuffer(make([]byte, 0)), - } - file.File = f - - _, err := io.Copy(file.fullContents, file.File) + err = tmpl.Execute(w, filters) if err != nil { - return FilteredFile{}, err + http.Error(w, err.Error(), http.StatusInternalServerError) } - contents := file.fullContents.String() - for key, value := range filter { - key = "@" + key + "@" - contents = strings.ReplaceAll(contents, key, value) - } - file.fullContents = bytes.NewBufferString(contents) - - return file, nil -} - -func (ff FilteredFile) Read(p []byte) (n int, err error) { - return ff.fullContents.Read(p) + // ok, tmpl has written the response } diff --git a/cmd/converge/server.go b/cmd/converge/server.go index 9feac0f..bb65d33 100644 --- a/cmd/converge/server.go +++ b/cmd/converge/server.go @@ -9,8 +9,6 @@ import ( "net/http" "os" "regexp" - "strings" - "sync" ) func parsePublicId(path string) (publicId string, _ error) { @@ -27,38 +25,6 @@ func catchAllHandler(w http.ResponseWriter, r *http.Request) { return } -type FileHandlerFilter struct { - http.Handler -} - -func (handler FileHandlerFilter) ServeHTTP(w http.ResponseWriter, r *http.Request) { - mutex := sync.Mutex{} - // pedantic - mutex.Lock() - - if r.TLS == nil { - filters["secure"] = "" - } else { - filters["secure"] = "s" - } - for _, header := range []string{"X-Forwarded-Proto", "X-Scheme", "X-Forwarded-Scheme"} { - values := r.Header.Values(header) - for _, value := range values { - if strings.ToLower(value) == "https" { - filters["secure"] = "s" - } - } - } - filters["host"] = r.Host - mutex.Unlock() - - w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate") - w.Header().Set("Pragma", "no-cache") - w.Header().Set("Expires", "0") - - handler.Handler.ServeHTTP(w, r) -} - func main() { downloadDir := "downloads" @@ -97,15 +63,12 @@ func main() { }, } + // websocket endpoints http.HandleFunc("/agent/", registrationService.Handle) http.HandleFunc("/client/", clientService.Handle) - filesystem := http.Dir(downloadDir) - filteredFilesystem := FilteredFileSystem{ - fs: filesystem, - } - fileHandler := FileHandlerFilter{http.FileServer(filteredFilesystem)} - + // create filehandler with templating for html files. + fileHandler := NewFileHandler(downloadDir) http.Handle("/docs/", http.StripPrefix("/docs/", fileHandler)) http.HandleFunc("/", catchAllHandler) diff --git a/static/index.html b/static/index.html index acffb94..2295fe9 100644 --- a/static/index.html +++ b/static/index.html @@ -12,136 +12,168 @@ -
+
+
+
-

About

+
-

- Converge is a utility for troubleshooting builds on continuous integration servers. - It solves a common problem where the cause of job failure is difficult to determine. - This is complicated further by the fact that build jobs are usually run on a build - farm where there is no access to the build agents or in more modern envrionments when - jobs are run in ephemeral containers. -

+

About

-

- With Converge it is possible to get remote shell access to such jobs. This works - by configuring the build job to connect to a Converge server using an agent program. - The agent program can be downloaded from within the CI job using curl or wget. - Next, an end-user can connect to the Converge server, a rendez-vous server, that connects - the client and server together. -

+

+ Converge is a utility for troubleshooting builds on continuous integration servers. + It solves a common problem where the cause of job failure is difficult to determine. + This is complicated further by the fact that build jobs are usually run on a build + farm where there is no access to the build agents or in more modern envrionments when + jobs are run in ephemeral containers. +

-

- The setup is such that the connection from client (end-user) to server (agent on CI job) - is end-to-end encrypted. The Converge server itself is no more than a bitpipe which pumps - data between client and agent. -

+

+ With Converge it is possible to get remote shell access to such jobs. This works + by configuring the build job to connect to a Converge server using an agent program. + The agent program can be downloaded from within the CI job using curl or wget. + Next, an end-user can connect to the Converge server, a rendez-vous server, that connects + the client and server together. +

-

- Both ssh and sftp are supported. Multiple shells are also allowed. -

+

+ The setup is such that the connection from client (end-user) to server (agent on CI job) + is end-to-end encrypted. The Converge server itself is no more than a bitpipe which pumps + data between client and agent. +

-

- There is a timeout mechanism in the agent such that jobs do not hang indefinitely waiting - for a connection. This mechanism is useful to make sure build agents do not keep - build agents occupied for a long time. By default, the agent exits with status 0 when - the first client exits after logging in. This behavior as well as general expiry can be - controlled from within a shell session by touching a .hold file. After logging in, the - user can control expiry of the session as instructed by messages in the ssh session. - When the timeout of a session is near the user is informed about this with messages - in the shell. -

+

+ Both ssh and sftp are supported. Multiple shells are also allowed. +

-

Usage

+

+ There is a timeout mechanism in the agent such that jobs do not hang indefinitely waiting + for a connection. This mechanism is useful to make sure build agents do not keep + build agents occupied for a long time. By default, the agent exits with status 0 when + the first client exits after logging in. This behavior as well as general expiry can be + controlled from within a shell session by touching a .hold file. After logging in, the + user can control expiry of the session as instructed by messages in the ssh session. + When the timeout of a session is near the user is informed about this with messages + in the shell. +

-

Continous integration jobs

+

Usage

-

- In a continous integration job, download the agent, chmod it and run it. -

- 
-     curl http@secure@://@host@/docs/agent > agent
+                
Continous integration jobs

+
+                

+                    In a continous integration job, download the agent, chmod it and run it.
+                

+                
+     # linux
+     curl http{{.secure}}://{{.host}}/docs/agent > agent
      chmod 755 agent
-     ./agent ws@secure@://@host@/agent/ID
+     ./agent ws{{.secure}}://{{.host}}/agent/ID
+
+     # windows
+     curl http{{.secure}}://{{.host}}/docs/agent.exe > agent.exe
+     agent ws{{.secure}}://{{.host}}/agent/ID
     

-    

-        Above, ID is a unique id
-        for the job. This should not conflict with other ids.
+                

+                    Above, ID is a unique id
+                    for the job. This should not conflict with other ids.
 
-        This connects the agent to the converge server. Clients can now connect to the Converge
-        server to establish a connection to the CI job through converge.
-    

+                    This connects the agent to the converge server. Clients can now connect to the Converge
+                    server to establish a connection to the CI job through converge.
+                

 
-    
Local clients: using ssh proxy command 

+                

+                    NOTE: When running the agent on windows, an exit of the remote session using
+                    exit in powershell or command prompt does not terminate the shell completely.
+                    To terminate the client ssh session must be killed by closing the terminal.
+                    Cleanup of remote processes on the agent appears to work properly despite this
+                    problem. It is just that exit of the windows shell (powershell or command prompt)
+                    is not detected properly.
+                

 
-    
-    curl http@secure@://@host@/docs/wsproxy > wsproxy
-    chmod 755 wsproxy
+                
Local clients: using ssh proxy command 

+
+                
wsproxy is a command that can be used as a proxy command for SSH which performs the connection to the
+                    remote server. This command needs to be downloaded only once (see downloads below). It does not depend on
+                    the converge implementation but only on the websocket standards. Other tools that
+                    provide a mapping of stdio to a websocket can also be used instead of wsproxy.
+                

+                

+                    Next step is to run a local SSH or SFTP client:
+                

+
+                
+    ssh -oServerAliveInterval=10 -oProxyCommand="wsproxy ws{{.secure}}://{{.host}}/client/ID"  abc@localhost
+    sftp -oServerAliveInterval=10 -oProxyCommand="wsproxy ws{{.secure}}://{{.host}}/client/ID" abc@localhost
     

 
-    
This is a command that can be used as a proxy command for SSH which performs the connection to the remote
-        server.

+                

+                    abc is a fixed user defined by converge. It has a very exciting password.
+                

 
-    

-        Next step is to run a local SSH or SFTP client:
-    

+                
Local clients: with a local TCP forwarding proxy

 
-    
-    ssh -oServerAliveInterval=10 -oProxyCommand="wsproxy ws@secure@://@host@/client/ID"  abc@localhost
-    sftp -oServerAliveInterval=10 -oProxyCommand="wsproxy ws@secure@://@host@/client/ID" abc@localhost
-    

+                

+                This option is less convenient than the proxy command because it requires two separate
+                commands to execute.
+                

 
-    

-        abc is a fixed user defined by converge. It has a very exciting password.
-    

+                

+                    Local clients can connect using regular ssh and sftp commands through a tunnel that
+                    translates a local TCP port to a websocket connection in converge. See
+                    the downloads section.
+                    This runs a local client that allows SSH to port 10000 and connects to converge using
+                    a websocket connection.
+                

 
-    
Local clients: with a local TCP forwarding proxy

 
-    This option is less convenient than the proxy command because it requires two separate
-    commands to execute.
+                

+                    Next step is to run a local SSH of SFTP client:
+                

 
-    

-        Local clients can connect using regular ssh and sftp commands through a tunnel that
-        translates a local TCP port to a websocket connection in converge.
-
-        First step is to download the tcptows program (see below):
-    

-
-    
-    curl http@secure@://@host@/docs/tcptows > tcptows
-    chmod 755 tcptows
-    ./tcptows 10000 ws@secure@://@host@/client/ID
-    

-
-    

-        This runs a local client that allows SSH to port 10000 and connects to converge using
-        a websocket connection.

-
-    

-        Next step is to run a local SSH of SFTP client:
-    

-
-    
+                
     ssh -oServerAliveInterval=10 -p 10000 abc@localhost
     sftp -oServerAliveInterval=10 -oPort=10000 abc@localhost
     

 
-    

-        abc is a fixed user defined by converge. It has a very exciting password.
-    

+                

+                    abc is a fixed user defined by converge. It has a very exciting password.
+                

 
-    
Downloads

+                
Downloads

 
-    
    
-        
  • agent: The agent to run inside aa CI job
-        
    • 
-        
  • tcptows: TCP to WS tunnel for allowing regular
-            SSH and SFTP clients to connect to converge.
-        
    • 
-        
  • wsproxy: SSH proxy command that can be directly used by ssh
-        
    • 
-    

+                
+                    
+                    
+                        
+                        
+                        
+                        
+                    
+                    
+                    
+                        
+                        
+                        
+                        
+                    
+                    
+                        
+                        
+                        
+                        
+                    
+                    
+                        
+                        
+                        
+                        
+                    
+                
ComponentPurposeLinuxWindows
agentThe agent to run inside aa CI jobagentagent.exe
wsproxySSH proxy command that can be directly used by sshwsproxywsproxy.exe
tcptowsTCP to WS tunnel for allowing regular
+                            SSH and SFTP clients to connect to convergetcptowstcptows.exe

+
+
+