Lots of work on docuemtation. The docs page now shows the correct

installation dependent URLs. For now using ServerALiveInterval
to avoid disconnects.

cmd/agent/agent.go

@@ -92,7 +92,7 @@ func sshServer(hostKeyFile string) *ssh.Server {
 		}
 	})
 
-	log.Println("starting ssh server")
+	log.Println("starting ssh server, waiting for debug sessions")
 	server := ssh.Server{
 		PasswordHandler: passwordAuth,
 		SubsystemHandlers: map[string]ssh.SubsystemHandler{
@@ -164,6 +164,8 @@ func main() {
 		log.Println("WebSocket connection error:", err)
 		return
 	}
+	conn.SetReadDeadline(time.Time{})
+	conn.SetWriteDeadline(time.Time{})
 	wsConn := websocketutil.NewWebSocketConn(conn)
 	defer wsConn.Close()
 

cmd/converge/server.go

@@ -1,14 +1,18 @@
 package main
 
 import (
+	"bytes"
 	"cidebug/pkg/converge"
 	"cidebug/pkg/websocketutil"
 	"fmt"
+	"io"
 	"log"
 	"net"
 	"net/http"
 	"os"
 	"regexp"
+	"strings"
+	"sync"
 )
 
 func parsePublicId(path string) (publicId string, _ error) {
@@ -25,6 +29,90 @@ func catchAllHandler(w http.ResponseWriter, r *http.Request) {
 	return
 }
 
+// filters to filter html content.
+var filters map[string]string = make(map[string]string)
+
+type FileHandlerFilter struct {
+	http.Handler
+}
+
+func (handler FileHandlerFilter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	mutex := sync.Mutex{}
+	// pedantic
+	mutex.Lock()
+
+	if r.TLS == nil {
+		filters["secure"] = ""
+	} else {
+		filters["secure"] = "s"
+	}
+	for _, header := range []string{"X-Forwarded-Proto", "X-Scheme", "X-Forwarded-Scheme"} {
+		values := r.Header.Values(header)
+		for _, value := range values {
+			if strings.ToLower(value) == "https" {
+				filters["secure"] = "s"
+			}
+		}
+	}
+	filters["host"] = r.Host
+	mutex.Unlock()
+
+	w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
+	w.Header().Set("Pragma", "no-cache")
+	w.Header().Set("Expires", "0")
+
+	handler.Handler.ServeHTTP(w, r)
+}
+
+type FilteredFileSystem struct {
+	fs http.FileSystem
+}
+
+func (ffs FilteredFileSystem) Open(name string) (http.File, error) {
+	log.Println("Name : " + name)
+	f, err := ffs.fs.Open(name)
+	if err != nil {
+		return nil, err
+	}
+	if !strings.HasSuffix(name, ".html") {
+		return f, nil
+	}
+	return NewFilteredFile(f, filters)
+}
+
+type FilteredFile struct {
+	http.File
+
+	// Read bytes 0..(index-1) already
+	index        int
+	fullContents *bytes.Buffer
+}
+
+func NewFilteredFile(f http.File, filter map[string]string) (FilteredFile, error) {
+	file := FilteredFile{
+		index:        0,
+		fullContents: bytes.NewBuffer(make([]byte, 0)),
+	}
+	file.File = f
+
+	_, err := io.Copy(file.fullContents, file.File)
+	if err != nil {
+		return FilteredFile{}, err
+	}
+	contents := file.fullContents.String()
+	for key, value := range filter {
+		key = "@" + key + "@"
+		contents = strings.ReplaceAll(contents, key, value)
+	}
+	file.fullContents = bytes.NewBufferString(contents)
+
+	return file, nil
+}
+
+func (ff FilteredFile) Read(p []byte) (n int, err error) {
+	return ff.fullContents.Read(p)
+}
+
 func main() {
 
 	downloadDir := "downloads"
@@ -65,7 +153,14 @@ func main() {
 
 	http.HandleFunc("/agent/", registrationService.Handle)
 	http.HandleFunc("/client/", clientService.Handle)
-	http.Handle("/docs/", http.StripPrefix("/docs/", http.FileServer(http.Dir(downloadDir))))
+
+	filesystem := http.Dir(downloadDir)
+	filteredFilesystem := FilteredFileSystem{
+		fs: filesystem,
+	}
+	fileHandler := FileHandlerFilter{http.FileServer(filteredFilesystem)}
+
+	http.Handle("/docs/", http.StripPrefix("/docs/", fileHandler))
 	http.HandleFunc("/", catchAllHandler)
 
 	// Start HTTP server

cmd/tcptows/tcptows.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"net"
 	"os"
+	"time"
 )
 
 func closeConnection(conn net.Conn) {
@@ -25,6 +26,8 @@ func handleConnection(conn net.Conn, wsURL string) {
 		log.Println("WebSocket connection error:", err)
 		return
 	}
+	_wsConn.SetReadDeadline(time.Time{})
+	_wsConn.SetWriteDeadline(time.Time{})
 	wsConn := websocketutil.NewWebSocketConn(_wsConn)
 	defer wsConn.Close()
 

cmd/wsproxy/proxy.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"net"
 	"os"
+	"time"
 )
 
 func closeConnection(conn net.Conn) {
@@ -29,6 +30,8 @@ func main() {
 	wsURL := os.Args[1]
 
 	_wsConn, _, err := websocket.DefaultDialer.Dial(wsURL, nil)
+	_wsConn.SetReadDeadline(time.Time{})
+	_wsConn.SetWriteDeadline(time.Time{})
 	if err != nil {
 		log.Println("WebSocket connection error:", err)
 		panic(err)

pkg/agent/help.txt

@@ -1,4 +1,4 @@
-Session is set to expire at %s
+Session is set to expire at %v
 
 The session expires automatically after %d time.
 If there are no more sessions after logging out, the agent
@@ -8,6 +8,6 @@ You can extend this time using
 
   touch $agentdir/.hold
 
-To prevent the agent from exiting after the last sessioni is gone,
+To prevent the agent from exiting after the last session is gone,
 also use the above command in any shell.
 

pkg/agent/session.go

@@ -144,9 +144,9 @@ func fileExists(filename string) bool {
 func (state *AgentState) expiryTime(filename string) time.Time {
 	stats, err := os.Stat(filename)
 	if err != nil {
-		return state.startTime
+		return state.startTime.Add(state.agentExpriryTime)
 	}
-	return stats.ModTime()
+	return stats.ModTime().Add(state.agentExpriryTime)
 }
 
 // Behavior to implement
@@ -177,6 +177,11 @@ func check() {
 			PrintHelpMessage(session.sshSession)
 		}
 	}
+
+	if !fileExists(holdFilename) && len(state.sessions) == 0 {
+		log.Printf("All clients disconnected and no '%s' file found, exiting", holdFilename)
+		os.Exit(0)
+	}
 }
 
 func messageUsers(message string) {

pkg/websocketutil/connections.go

@@ -72,6 +72,8 @@ func ConnectWebSocket(conn net.Conn, urlStr string) (net.Conn, error) {
 	if err != nil {
 		return nil, err
 	}
+	wsConn.SetReadDeadline(time.Time{})
+	wsConn.SetWriteDeadline(time.Time{})
 
 	return NewWebSocketConn(wsConn), nil
 }

pkg/websocketutil/services.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"time"
 )
 
 type WebSocketAddr string
@@ -28,6 +29,8 @@ func handleWebSocket(w http.ResponseWriter, r *http.Request,
 		log.Println("Error upgrading to WebSocket:", err)
 		return
 	}
+	conn.SetReadDeadline(time.Time{})
+	conn.SetWriteDeadline(time.Time{})
 	wsConn := NewWebSocketConn(conn)
 	defer wsConn.Close()
 

static/index.html

@@ -14,6 +14,39 @@
 
 <div class="container">
 
+    <h1>About</h1>
+
+    <p>
+    Converge is a utility for troubleshooting builds on continuous integration serves.
+    It solves a common problem where the cause of job failure is difficult to determine.
+    This is complicated furhter by the fact that build jobs are usually run on a build
+    farm where there is no access to the build agents or in more modern envrionments when
+    jobs are run in ephemeral containers.
+    </p>
+
+    <p>
+    With Converge it is possible to get remote shell access to such jobs. This works
+    by configuring the build job to connect to a Converge server using an agent program.
+    The agent program can be downloaded from within the CI job using curl or wget.
+    Next, an end-use can connect to the Converge server, a rendez-vous server, that connects
+    the client and server together.
+    </p>
+
+    <p>
+    The setup is such that the connection from client (end-user) to server (agent on CI job)
+    is end-to-end encrypted. The Converge server itself is no more than a bitpipe which pumps
+    data between client and agent.
+    </p>
+
+    <p>
+    Both ssh and sftp are supported. Multiple shells are also allowed.
+    </p>
+
+    <p>
+    There is a timeout mechanism in the agent such that jobs do not hang indefinetely waiting
+    for a connection.
+    </p>
+
     <h1>Usage</h1>
 
     <h2>Continous integration jobs</h2>
@@ -22,12 +55,12 @@
         In a continous integration job, download the agent, chmod it and run it.
     </p>
     <pre>
-     curl https://HOSTPORT/docs/agent > agent
-     ./agent wss://HOST:PORT/agent/ID
+     curl http@secure@://@host@/docs/agent > agent
      chmod 755 agent
+     ./agent ws@secure@://@host@/agent/ID
     </pre>
     <p>
-        Above, HOST:PORT is the hostname:port of the converge server and ID is a unique id
+        Above, ID is a unique id
         for the job. This should not conflict with other ids.
 
         This connects the agent to the converge server. Clients can now connect to converge
@@ -37,7 +70,7 @@
     <h2>Local clients: using ssh proxy command </h2>
 
     <pre>
-    curl https://HOST:PORT/docs/wsproxy > wsproxy
+    curl http@secure@://@host@/docs/wsproxy > wsproxy
     chmod 755 wsproxy
     </pre>
 
@@ -49,8 +82,8 @@
     </p>
 
     <pre>
-    ssh -oProxyCommand="wsproxy https://HOST:PORT/client/ID" -p 10000 abc@localhost
-    sftp -oProxyCommand="wsproxy https://HOST:PORT/client/ID" -oPort 10000 abc@localhost
+    ssh -oServerAliveInterval=10 -oProxyCommand="wsproxy ws@secure@://@host@/client/ID"  abc@localhost
+    sftp -oServerAliveInterval=10 -oProxyCommand="wsproxy ws@secure@://@host@/client/ID" abc@localhost
     </pre>
 
     <p>
@@ -70,15 +103,9 @@
     </p>
 
     <pre>
-    # for HTTP hosted server
-    curl http://HOST:PORT/docs/tcptows > tcptows
+    curl http@secure@://@host@/docs/tcptows > tcptows
     chmod 755 tcptows
-    ./tcptows 10000 ws://HOST:PORT/client/ID
-
-    # for HTTPS hosted server
-    curl https://HOST:PORT/docs/tcptows > tcptows
-    chmod 755 tcptows
-    ./tcptows 10000 wss://HOST:PORT/client/ID
+    ./tcptows 10000 ws@secure@://@host@/client/ID
     </pre>
 
     <p>
@@ -90,8 +117,8 @@
     </p>
 
     <pre>
-    ssh -p 10000 abc@localhost
-    sftp -oPort 10000 abc@localhost
+    ssh -oServerAliveInterval=10 -p 10000 abc@localhost
+    sftp -oServerAliveInterval=10 -oPort 10000 abc@localhost
     </pre>
 
     <p>