Lots of work on making easier interactive documentation, especially to make working with SSH public keys really easy.

Next step is to do more validation in the UI.
Specifically:
* validate authorized keys
* detection of accidental use of a private key

Then, password based access can be disabled.

Dockerfile

@@ -28,4 +28,4 @@ COPY --from=builder /opt/converge/bin/agent \
                     /opt/converge/static/
 COPY --from=builder /opt/converge/static/ /opt/converge/static/
 
-ENTRYPOINT ["/opt/converge/bin/converge", "-d", "/opt/converge/static" ]
+ENTRYPOINT ["/opt/converge/bin/converge", "-s", "/opt/converge/static", "-d", "/opt/converge/static" ]

cmd/converge/converge.go

@@ -48,14 +48,16 @@ func printHelp(msg string) {
 		"an embedded SSH server to provide interactive access to the end-user. This works\n" +
 		"both on linux and on windows.\n" +
 		"\n" +
-		"-d <contentdir>: directory where static content of converge is placed"
+		"-s <contentdir>: directory where static content of converge is placed\n" +
+		"-d <downloaddir>: directory where downloads of converge are placed\n"
 	fmt.Fprintln(os.Stderr, helpText)
 	os.Exit(1)
 }
 
 func main() {
 
-	downloadDir := "../static"
+	downloaddir := "."
+	staticdir := "../static"
 
 	args := os.Args[1:]
 	for len(args) > 0 && strings.HasPrefix(args[0], "-") {
@@ -64,14 +66,20 @@ func main() {
 			if len(args) <= 1 {
 				printHelp("The -d option expects an argument")
 			}
-			downloadDir = args[1]
+			downloaddir = args[1]
+			args = args[1:]
+		case "-s":
+			if len(args) <= 1 {
+				printHelp("The -s option expects an argument")
+			}
+			staticdir = args[1]
 			args = args[1:]
 		default:
 			printHelp("Unknown option " + args[0])
 		}
 		args = args[1:]
 	}
-	log.Println("Content directory", downloadDir)
+	log.Println("Content directory", staticdir)
 
 	if len(args) != 0 {
 		printHelp("")
@@ -158,7 +166,9 @@ func main() {
 	// create filehandler with templating for html files.
 	http.Handle("/docs/", http.StripPrefix("/docs/", http.HandlerFunc(pageHandler)))
 	http.Handle("/static/", http.StripPrefix("/static/",
-		http.FileServer(http.Dir(downloadDir))))
+		http.FileServer(http.Dir(staticdir))))
+	http.Handle("/downloads/", http.StripPrefix("/downloads/",
+		http.FileServer(http.Dir(downloaddir))))
 	http.HandleFunc("/", catchAllHandler)
 
 	// create usage generator

cmd/converge/usage.go

@@ -2,16 +2,14 @@ package main
 
 import (
 	"converge/pkg/server/templates"
-	"log"
 	"math/rand"
 	"net/http"
 	"os"
 	"strconv"
+	"strings"
 )
 
 func generateCLIExammple(w http.ResponseWriter, r *http.Request) {
-	log.Println("usage: got ", r.URL.Path)
-
 	err := r.ParseForm()
 	if err != nil {
 		http.Error(w, "Error parsing form", http.StatusBadRequest)
@@ -27,14 +25,17 @@ func generateCLIExammple(w http.ResponseWriter, r *http.Request) {
 	}
 	remoteShells := r.Form["remote-shell"]
 	localShells := r.Form["local-shell"]
-	keys := r.FormValue("ssh-keys")
+	keysString := r.FormValue("ssh-keys")
-	log.Printf("remote_shells %v", remoteShells)
+	sshPublicKeys := make([]string, 0)
-	log.Printf("local_shells %v", localShells)
+	for _, line := range strings.Split(keysString, "\n") {
-	log.Printf("ssh-keys %v", keys)
+		line := strings.TrimSpace(line)
-
+		if line != "" {
+			sshPublicKeys = append(sshPublicKeys, line)
+		}
+	}
 	access := getConvergeAccess(r, getAgentSshUser())
 
-	usageInputs := templates.NewUsageInputs(id, remoteShells, localShells)
+	usageInputs := templates.NewUsageInputs(id, sshPublicKeys, remoteShells, localShells)
 	err = templates.ShellUsage(access, usageInputs).Render(r.Context(), w)
 	if err != nil {
 		http.Error(w, err.Error(), 500)

pkg/server/templates/about.templ

@@ -19,55 +19,123 @@ templ About() {
           the client and server together.
       </p>
 
+      <h2>other tools</h2>
+
+      <p>Using available existing tools such as
+          <a href="https://github.com/namespacelabs/breakpoint">breakpoint</a> in combination
+          with a websocket tunneling tool such as
+          <a href="https://github.com/erebe/wstunnel">wstunnel</a> a similar solution can be
+          obtained. There are however some problems with these solutions that converge is
+          trying to address:
+      </p>
+
+      <p>
+      <ul>
+      <li>deployment: Breakpoint uses an embedded SSH server which is a really good idea but
+      uses the QUIC protocol for connecting to a rendez-vous server. The rendez-vous server than
+      exposes a random port for every client. This make deployment on kubernetes really hard
+      where fixed ports must be used and QUIC is also not a widely supported protocol.</li>
+      <li>The problem with the random ports can be solved by using wstunnel running together
+      with breakpoint server in a kubernetes pod, where wstunnel can forward traffic over an
+      extern websocket connection to the local random port that breakpoint server is listening on.</li>
+      <li>breakpoint leaves it open on how users install the breakpoint executable (agent). </li>
+      <li>Because of the hacky nature of this setup, it is very difficult for users to use
+      and troubleshoot when things go wrong. </li>
+      </ul>
+
+      </p>
+      Converve server addresses these issues in the following ways:
+      <ul>
+      <li>Use the websocket protocol both for agents and for clients, providing a fixed port and
+          a supported protocol for kubernetes deploymment.</li>
+      <li>Providing online documentation where the instructions take into account the
+          hostname and protocol where converge is running allowing users to cut and paste
+          instructions that can be used without modification. In the usage page the users
+          can even generate the correct agent startup commands and client connection commands
+          based on the type of shell they are connecting to. </li>
+      <li>Converge server provides out of the box downloads of required software. This makes sure
+          client and server are always up to date. In addition a protocol version check is done. </li>
+      <li>User-friendly error messages can be given to users in most case when things do not work
+          out because of <code>wsproxy</code>, an SSH proxy command that also talk to the server
+          to tell the user if a connection is accepted and if not why not. </li>
+      <li>A live screen showing the current sessions that are running. </li>
+      <li>Interactivity in the user's session with notifications about timeouts and a very
+      simple inactivity timmeout mechanism. </li>
+      <li>Possibility for the user to define his own shell. </li>
+      <li>Support for unix like bash shells and command prompt and powershell. </li>
+      </ul>
+      <p>
+      </p>
+
+
+      <h2>how it works</h2>
+
+      <p>
+          The steps involved are as follows:
+          <ul>
+          <li>The agent connects to converge server. If no id is specified than a new id will
+              be generated. The ids specified by different agents must be unique. If the agent
+              specifies an id that is already in use, then a new id will be generated.
+              When started the agent will echo the commands to connect to it in its output.
+          </li>
+           <li>
+              Since the emmbedded SSH server in the agent will allow multiple clients to connect
+              to it, it wants to listen for copnnections. By default it cannot do this, it just
+              setup a connection to the converge server, but the converge server can in general
+              not connect back to it because of networking. Therefore, a multiplexing library is
+              used to establish multiple virtual connections over a single TCP connection.
+              The agent can now listen for connections from clients.
+          </li>
+          <li>The agent connects to the converge server using the commmand specified by the agent.
+              The converge server can then match the agent with the client based on the id and
+              the connectio at network level is established.
+          </li>
+          <li>The embedded SSH server now performs authentication, after successful login,
+              a shell is spwaned and the network connection of the user is connected to it.
+              The connection is practically identical to a regular terminal connection. To
+              achieve this some magic is used to make the shell beiieve it is connected to a
+              terminal.
+          </li>
+          </ul>
+      </p>
+
+      <h2>Security</h2>
+
       <p>
           The setup is such that the connection from client (end-user) to server (agent on CI job)
           is end-to-end encrypted. The Converge server itself is no more than a bitpipe which pumps
           data between client and agent.
       </p>
 
+      <p>Currently converge server still supports password based login but this will be disabled.
+         Image two people configuring an agent with the same id where one of the agents actually
+         gets it and other gets a new id. Now, with a password each user can access each other's
+         agents. This is of course highly confusing and undesirable. Converge server already support
+         authorized keys but this is not yet mandatory. I is made extremely easy through the
+         <a href="usage.html">usage</a> page to configure this, so the additional complexity should
+         not be an issue.
+      </p>
+
+      <h2>SSH and SFTP</h2>
+
       <p>
           Both ssh and sftp are supported. Multiple shells are also allowed.
       </p>
 
+      <h2>Timeouts</h2>
+
       <p>
           There is a timeout mechanism in the agent such that jobs do not hang indefinitely
           waiting for a connection. This mechanism is useful to make sure build agents do not keep
           build agents occupied for a long time. By default, the agent exits with status 0 when
-          the first client exits after logging in. This behavior as well as general expiry can be
+          the first client exits after logging in.
-          controlled from within a shell session by touching a .hold file. After logging in, the
-          user can control expiry of the session as instructed by messages in the ssh session.
-          When the timeout of a session is near the user is informed about this with messages
-          in the shell.
       </p>
-
+      <p>When the user touches a .hold file, the agent keeps waiting for connections even
-      <p>end-to-end encryoption</p>
+         after the last client logs out, taking into account the timeout.
-      <p> ssh keys</p>
-      <p>agent options </p>
-      <p>client access </p>
-
-      <h2>Local clients: using ssh with a proxy command </h2>
-
-      <p>
-        <code>wsproxy</code> is a command that can be used as a proxy command for SSH which performs the connection to the
-        remote server. This command needs to be downloaded only once (see <a href="downloads.html">downloads</a>). It does not depend on
-        the converge implementation but only on the websocket standards. Other tools that
-        provide a mapping of stdio to a websocket can also be used instead of wsproxy.
       </p>
+      <p>The sessions have an inactivity timeout. Any keypress on the keyboard by a user
+      is interpreted as activity. </p>
 
-      <h2>Local clients: using SSH with a local TCP forwarding proxy</h2>
-
-      <p>
-        This option is less convenient than the proxy command because it requires two separate
-        commands to execute.
-      </p>
-
-      <p>
-         Local clients can connect using regular ssh and sftp commands through a tunnel that
-         translates a local TCP port to a websocket connection in converge. See
-         the <a href="downloads.html">downloads</a> section.
-         This runs a local client that allows SSH to port 10000 and connects to converge using
-         a websocket connection.
-     </p>
 
      <h2>Remote shell usage</h2>
 

pkg/server/templates/downloads.templ

@@ -17,20 +17,20 @@ templ Downloads() {
                         </thead>
                         <tr>
                             <td>agent</td>
-                            <td><a href="../static/agent">agent</a></td>
+                            <td><a href="../downloads/agent">agent</a></td>
-                            <td><a href="../static/agent.exe">agent.exe</a></td>
+                            <td><a href="../downloads/agent.exe">agent.exe</a></td>
                             <td>The agent to run inside aa CI job</td>
                         </tr>
                         <tr>
                             <td>wsproxy</td>
-                            <td><a href="../static/wsproxy">wsproxy</a></td>
+                            <td><a href="../downloads/wsproxy">wsproxy</a></td>
-                            <td><a href="../static/wsproxy.exe">wsproxy.exe</a></td>
+                            <td><a href="../downloads/wsproxy.exe">wsproxy.exe</a></td>
                             <td>SSH proxy command that can be directly used by ssh</td>
                         </tr>
                         <tr>
                             <td>tcptows</td>
-                            <td><a href="../static/tcptows">tcptows</a></td>
+                            <td><a href="../downloads/tcptows">tcptows</a></td>
-                            <td><a href="../static/tcptows.exe">tcptows.exe</a></td>
+                            <td><a href="../downloads/tcptows.exe">tcptows.exe</a></td>
                             <td>You typically do not need to download this.
                                 It was used for testing at the beginning and can still be used
                                 as a generic TCP to WS tunnel for allowing regular

pkg/server/templates/usage.templ

@@ -12,16 +12,23 @@ templ AgentUsage(access models.ConvergeAccess, shells map[string]bool, usageInpu
     </p>
 
     if shells[BASH] {
-        <pre>{`
+        <pre>{addSshKeys(BASH, usageInputs.SshKeys)}
-        `}curl http{access.Secure}://{access.HostPort}/static/agent > agent{`
+        curl --fail-with-body http{access.Secure}://{access.HostPort}/downloads/agent > agent{`
         chmod 755 agent
         `}./agent --id {usageInputs.Id} ws{access.Secure}://{access.HostPort}{`
         rm -f agent
         `}</pre>
     }
-    if shells[CMD] || shells[POWERSHELL]  {
+    if shells[CMD]  {
-        <pre>{`
+        <pre>{addSshKeys(CMD, usageInputs.SshKeys)}
-        `}curl http{access.Secure}://{access.HostPort}/static/agent.exe > agent.exe{`
+        curl --fail-with-body http{access.Secure}://{access.HostPort}/downloads/agent.exe > agent.exe{`
+        `}agent --id {usageInputs.Id} ws{access.Secure}://{access.HostPort}{`
+        del agent.exe
+        `}</pre>
+    }
+    if shells[POWERSHELL]  {
+        <pre>{addSshKeys(POWERSHELL, usageInputs.SshKeys)}
+        curl --fail-with-body http{access.Secure}://{access.HostPort}/downloads/agent.exe > agent.exe{`
         `}agent --id {usageInputs.Id} ws{access.Secure}://{access.HostPort}{`
         del agent.exe
         `}</pre>
@@ -41,11 +48,41 @@ templ AgentUsage(access models.ConvergeAccess, shells map[string]bool, usageInpu
 
     <h2>Connecting to the agent</h2>
 
+    <p>The embedded ssh server in the agent supports both ssh and sftp. The user name is fixed
+       at <code>{ access.Username }</code>. This is the user used to connect to the embedded
+       SSH server, after logging in however you will be running in a shell that is started
+       by the same user that started the agent.
+    </p>
+
      <pre>{`
           `}ssh -oServerAliveInterval=10 -oProxyCommand="wsproxy ws{access.Secure}://{access.HostPort}/client/{usageInputs.Id}"  { access.Username }{"@localhost"}   {`
           `}sftp -oServerAliveInterval=10 -oProxyCommand="wsproxy ws{access.Secure}://{access.HostPort}/client/{usageInputs.Id}" { access.Username }{"@localhost"}   {`
           `}</pre>
 
+    <p>This requires the <code>wsproxy</code> utility which is available in the
+       <a href="downloads.html">downloads</a> section. This utility must be downloaded
+       only once since it is quite generic. It will warn you when it a newer version must
+       be downloaded.
+    </p>
+
+    <p>For other ssh clients that do not support the openssh ProxyCommand option, there is another
+    way to connect. In this method, a local port forwarder is started that forwards a local port
+    to the webserver. Then you can start an ssh client that connects to the local tcp port.
+    </p>
+    <pre>{`
+         `}ssh -oServerAliveInterval=10 -p 10000  { access.Username }{"@localhost"}   {`
+         `}sftp -oServerAliveInterval=10 -p 10000 { access.Username }{"@localhost"}   {`
+         `}</pre>
+
+    <p>This requires the <code>tcptows</code> utility which is available in the
+       <a href="downloads.html">downloads</a> section. The utility must be started beforehand
+       using:
+    </p>
+    <pre>{`
+         `}tcptows ws{access.Secure}://{access.HostPort}/client/{usageInputs.Id}   {`
+         `}tcptows ws{access.Secure}://{access.HostPort}/client/{usageInputs.Id}   {`
+         `}</pre>
+
     <h2>Working with the agent</h2>
 
      if shells[BASH] {
@@ -90,25 +127,11 @@ templ AgentUsage(access models.ConvergeAccess, shells map[string]bool, usageInpu
     </div>
 }
 
-templ LocalShellUsage(access models.ConvergeAccess, shells map[string]bool, usageInput UsageInputs) {
-  <div>
-  if shells[BASH] {
-     <p>bash</p>
-  }
-  if shells[CMD] {
-  <p>cmd</p>
-  }
-  if shells[POWERSHELL] {
-  <p>powershell</p>
-  }
-  </div>
-}
 
 templ ShellUsage(access models.ConvergeAccess, usageInputs UsageInputs) {
   <div>
   @AgentUsage(access, usageInputs.RemoteShells, usageInputs)
   </div>
-  @LocalShellUsage(access, usageInputs.LocalShells, usageInputs)
 }
 
 
@@ -163,25 +186,6 @@ templ Usage(access models.ConvergeAccess) {
 
       <div id="example-cli">
       </div>
-
-
-
-
-
-                      <pre>                                         {`
-          `}# linux                                                 {`
-          `}echo "ssh-rsa dkddkdkkk a@b.c" > .authorized_keys       {`
-          `}echo "ssh-rsa adfadjfdf d@e.f" >> .authorized_keys      {`
-          `}                                                        {`
-          `}# windows                                               {`
-          `}echo ssh-rsa dkddkdkkk a@b.c > .authorized_keys         {`
-          `}echo ssh-rsa adfadjfdf d@e.f >> .authorized_keys
-                      </pre>
-                      <p>
-                          Note that on windows you should not used quotes.
-                      </p>
-
-
     <script>
     function setCookie(name, value, days) {
         let expires = "";
@@ -274,8 +278,8 @@ templ Usage(access models.ConvergeAccess) {
     document.getElementById('inputs').addEventListener('change', saveFormToCookie);
 
     document.body.addEventListener('htmx:load', function(event) {
-      //console.log("htmx:load")
+      console.log("htmx:load")
-      //loadFormFromCookie();
+      loadFormFromCookie();
     });
 
     document.body.addEventListener('htmx:afterSettle', function(event) {

pkg/server/templates/usageinputs.go

@@ -1,5 +1,9 @@
 package templates
 
+import (
+	"fmt"
+)
+
 type UsageInputs struct {
 	Id           string
 	SshKeys      []string
@@ -7,9 +11,10 @@ type UsageInputs struct {
 	LocalShells  map[string]bool
 }
 
-func NewUsageInputs(id string, remoteShells []string, localShells []string) UsageInputs {
+func NewUsageInputs(id string, sshPublicKeys []string, remoteShells []string, localShells []string) UsageInputs {
 	inputs := UsageInputs{
 		Id:           id,
+		SshKeys:      sshPublicKeys,
 		RemoteShells: make(map[string]bool),
 		LocalShells:  make(map[string]bool),
 	}
@@ -21,3 +26,20 @@ func NewUsageInputs(id string, remoteShells []string, localShells []string) Usag
 	}
 	return inputs
 }
+
+func addSshKeys(shell string, keys []string) string {
+	quote := `"`
+	if shell == CMD {
+		quote = ""
+	}
+	res := ""
+	for index, key := range keys {
+		operator := ">>"
+		if index == 0 {
+			operator = ">"
+		}
+		res += fmt.Sprintf("        echo %s%s%s %s .authorized_keys\n", quote, key, quote,
+			operator)
+	}
+	return res + "       "
+}