fileserver now uses go template language.
updated docs for windows.
This commit is contained in:
parent
cd477b7b0a
commit
ff1c13cc98
@ -1,59 +1,67 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// filters to filter html content.
|
||||
var filters map[string]string = make(map[string]string)
|
||||
|
||||
type FilteredFileSystem struct {
|
||||
fs http.FileSystem
|
||||
type FileHandlerFilter struct {
|
||||
dir string
|
||||
fileHandler http.Handler
|
||||
}
|
||||
|
||||
func (ffs FilteredFileSystem) Open(name string) (http.File, error) {
|
||||
f, err := ffs.fs.Open(name)
|
||||
func NewFileHandler(dir string) *FileHandlerFilter {
|
||||
handler := FileHandlerFilter{
|
||||
dir: dir,
|
||||
fileHandler: http.FileServer(http.Dir(dir)),
|
||||
}
|
||||
return &handler
|
||||
}
|
||||
|
||||
func (handler FileHandlerFilter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
contextPath := r.URL.Path
|
||||
path := filepath.Join(handler.dir, contextPath)
|
||||
fileInfo, err := os.Stat(path)
|
||||
if err == nil && fileInfo.IsDir() {
|
||||
contextPath = filepath.Join(contextPath, "index.html")
|
||||
}
|
||||
|
||||
if !strings.HasSuffix(contextPath, ".html") {
|
||||
handler.fileHandler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
|
||||
w.Header().Set("Pragma", "no-cache")
|
||||
w.Header().Set("Expires", "0")
|
||||
|
||||
filters := make(map[string]string)
|
||||
if r.TLS == nil {
|
||||
filters["secure"] = ""
|
||||
} else {
|
||||
filters["secure"] = "s"
|
||||
}
|
||||
for _, header := range []string{"X-Forwarded-Proto", "X-Scheme", "X-Forwarded-Scheme"} {
|
||||
values := r.Header.Values(header)
|
||||
for _, value := range values {
|
||||
if strings.ToLower(value) == "https" {
|
||||
filters["secure"] = "s"
|
||||
}
|
||||
}
|
||||
}
|
||||
filters["host"] = r.Host
|
||||
|
||||
tmpl, err := template.ParseFiles(filepath.Join(handler.dir, contextPath))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
// let the filehandler generate the rror
|
||||
handler.fileHandler.ServeHTTP(w, r)
|
||||
}
|
||||
if !strings.HasSuffix(name, ".html") {
|
||||
return f, nil
|
||||
}
|
||||
return NewFilteredFile(f, filters)
|
||||
}
|
||||
|
||||
type FilteredFile struct {
|
||||
http.File
|
||||
|
||||
// Read bytes 0..(index-1) already
|
||||
index int
|
||||
fullContents *bytes.Buffer
|
||||
}
|
||||
|
||||
func NewFilteredFile(f http.File, filter map[string]string) (FilteredFile, error) {
|
||||
file := FilteredFile{
|
||||
index: 0,
|
||||
fullContents: bytes.NewBuffer(make([]byte, 0)),
|
||||
}
|
||||
file.File = f
|
||||
|
||||
_, err := io.Copy(file.fullContents, file.File)
|
||||
err = tmpl.Execute(w, filters)
|
||||
if err != nil {
|
||||
return FilteredFile{}, err
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
contents := file.fullContents.String()
|
||||
for key, value := range filter {
|
||||
key = "@" + key + "@"
|
||||
contents = strings.ReplaceAll(contents, key, value)
|
||||
}
|
||||
file.fullContents = bytes.NewBufferString(contents)
|
||||
|
||||
return file, nil
|
||||
}
|
||||
|
||||
func (ff FilteredFile) Read(p []byte) (n int, err error) {
|
||||
return ff.fullContents.Read(p)
|
||||
// ok, tmpl has written the response
|
||||
}
|
||||
|
||||
@ -9,8 +9,6 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
func parsePublicId(path string) (publicId string, _ error) {
|
||||
@ -27,38 +25,6 @@ func catchAllHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
type FileHandlerFilter struct {
|
||||
http.Handler
|
||||
}
|
||||
|
||||
func (handler FileHandlerFilter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
mutex := sync.Mutex{}
|
||||
// pedantic
|
||||
mutex.Lock()
|
||||
|
||||
if r.TLS == nil {
|
||||
filters["secure"] = ""
|
||||
} else {
|
||||
filters["secure"] = "s"
|
||||
}
|
||||
for _, header := range []string{"X-Forwarded-Proto", "X-Scheme", "X-Forwarded-Scheme"} {
|
||||
values := r.Header.Values(header)
|
||||
for _, value := range values {
|
||||
if strings.ToLower(value) == "https" {
|
||||
filters["secure"] = "s"
|
||||
}
|
||||
}
|
||||
}
|
||||
filters["host"] = r.Host
|
||||
mutex.Unlock()
|
||||
|
||||
w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
|
||||
w.Header().Set("Pragma", "no-cache")
|
||||
w.Header().Set("Expires", "0")
|
||||
|
||||
handler.Handler.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
||||
downloadDir := "downloads"
|
||||
@ -97,15 +63,12 @@ func main() {
|
||||
},
|
||||
}
|
||||
|
||||
// websocket endpoints
|
||||
http.HandleFunc("/agent/", registrationService.Handle)
|
||||
http.HandleFunc("/client/", clientService.Handle)
|
||||
|
||||
filesystem := http.Dir(downloadDir)
|
||||
filteredFilesystem := FilteredFileSystem{
|
||||
fs: filesystem,
|
||||
}
|
||||
fileHandler := FileHandlerFilter{http.FileServer(filteredFilesystem)}
|
||||
|
||||
// create filehandler with templating for html files.
|
||||
fileHandler := NewFileHandler(downloadDir)
|
||||
http.Handle("/docs/", http.StripPrefix("/docs/", fileHandler))
|
||||
http.HandleFunc("/", catchAllHandler)
|
||||
|
||||
|
||||
@ -12,136 +12,168 @@
|
||||
<script src="js/bootstrap.bundle.min.js"
|
||||
crossorigin="anonymous"></script>
|
||||
|
||||
<div class="container">
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
<div class="col">
|
||||
|
||||
<h1>About</h1>
|
||||
<div class="container">
|
||||
|
||||
<p>
|
||||
Converge is a utility for troubleshooting builds on continuous integration servers.
|
||||
It solves a common problem where the cause of job failure is difficult to determine.
|
||||
This is complicated further by the fact that build jobs are usually run on a build
|
||||
farm where there is no access to the build agents or in more modern envrionments when
|
||||
jobs are run in ephemeral containers.
|
||||
</p>
|
||||
<h1>About</h1>
|
||||
|
||||
<p>
|
||||
With Converge it is possible to get remote shell access to such jobs. This works
|
||||
by configuring the build job to connect to a Converge server using an agent program.
|
||||
The agent program can be downloaded from within the CI job using curl or wget.
|
||||
Next, an end-user can connect to the Converge server, a rendez-vous server, that connects
|
||||
the client and server together.
|
||||
</p>
|
||||
<p>
|
||||
Converge is a utility for troubleshooting builds on continuous integration servers.
|
||||
It solves a common problem where the cause of job failure is difficult to determine.
|
||||
This is complicated further by the fact that build jobs are usually run on a build
|
||||
farm where there is no access to the build agents or in more modern envrionments when
|
||||
jobs are run in ephemeral containers.
|
||||
</p>
|
||||
|
||||
<p>
|
||||
The setup is such that the connection from client (end-user) to server (agent on CI job)
|
||||
is end-to-end encrypted. The Converge server itself is no more than a bitpipe which pumps
|
||||
data between client and agent.
|
||||
</p>
|
||||
<p>
|
||||
With Converge it is possible to get remote shell access to such jobs. This works
|
||||
by configuring the build job to connect to a Converge server using an agent program.
|
||||
The agent program can be downloaded from within the CI job using curl or wget.
|
||||
Next, an end-user can connect to the Converge server, a rendez-vous server, that connects
|
||||
the client and server together.
|
||||
</p>
|
||||
|
||||
<p>
|
||||
Both ssh and sftp are supported. Multiple shells are also allowed.
|
||||
</p>
|
||||
<p>
|
||||
The setup is such that the connection from client (end-user) to server (agent on CI job)
|
||||
is end-to-end encrypted. The Converge server itself is no more than a bitpipe which pumps
|
||||
data between client and agent.
|
||||
</p>
|
||||
|
||||
<p>
|
||||
There is a timeout mechanism in the agent such that jobs do not hang indefinitely waiting
|
||||
for a connection. This mechanism is useful to make sure build agents do not keep
|
||||
build agents occupied for a long time. By default, the agent exits with status 0 when
|
||||
the first client exits after logging in. This behavior as well as general expiry can be
|
||||
controlled from within a shell session by touching a .hold file. After logging in, the
|
||||
user can control expiry of the session as instructed by messages in the ssh session.
|
||||
When the timeout of a session is near the user is informed about this with messages
|
||||
in the shell.
|
||||
</p>
|
||||
<p>
|
||||
Both ssh and sftp are supported. Multiple shells are also allowed.
|
||||
</p>
|
||||
|
||||
<h1>Usage</h1>
|
||||
<p>
|
||||
There is a timeout mechanism in the agent such that jobs do not hang indefinitely waiting
|
||||
for a connection. This mechanism is useful to make sure build agents do not keep
|
||||
build agents occupied for a long time. By default, the agent exits with status 0 when
|
||||
the first client exits after logging in. This behavior as well as general expiry can be
|
||||
controlled from within a shell session by touching a .hold file. After logging in, the
|
||||
user can control expiry of the session as instructed by messages in the ssh session.
|
||||
When the timeout of a session is near the user is informed about this with messages
|
||||
in the shell.
|
||||
</p>
|
||||
|
||||
<h2>Continous integration jobs</h2>
|
||||
<h1>Usage</h1>
|
||||
|
||||
<p>
|
||||
In a continous integration job, download the agent, chmod it and run it.
|
||||
</p>
|
||||
<pre>
|
||||
curl http@secure@://@host@/docs/agent > agent
|
||||
<h2>Continous integration jobs</h2>
|
||||
|
||||
<p>
|
||||
In a continous integration job, download the agent, chmod it and run it.
|
||||
</p>
|
||||
<pre>
|
||||
# linux
|
||||
curl http{{.secure}}://{{.host}}/docs/agent > agent
|
||||
chmod 755 agent
|
||||
./agent ws@secure@://@host@/agent/ID
|
||||
./agent ws{{.secure}}://{{.host}}/agent/ID
|
||||
|
||||
# windows
|
||||
curl http{{.secure}}://{{.host}}/docs/agent.exe > agent.exe
|
||||
agent ws{{.secure}}://{{.host}}/agent/ID
|
||||
</pre>
|
||||
<p>
|
||||
Above, ID is a unique id
|
||||
for the job. This should not conflict with other ids.
|
||||
<p>
|
||||
Above, ID is a unique id
|
||||
for the job. This should not conflict with other ids.
|
||||
|
||||
This connects the agent to the converge server. Clients can now connect to the Converge
|
||||
server to establish a connection to the CI job through converge.
|
||||
</p>
|
||||
This connects the agent to the converge server. Clients can now connect to the Converge
|
||||
server to establish a connection to the CI job through converge.
|
||||
</p>
|
||||
|
||||
<h2>Local clients: using ssh proxy command </h2>
|
||||
<p>
|
||||
NOTE: When running the agent on windows, an exit of the remote session using
|
||||
exit in powershell or command prompt does not terminate the shell completely.
|
||||
To terminate the client ssh session must be killed by closing the terminal.
|
||||
Cleanup of remote processes on the agent appears to work properly despite this
|
||||
problem. It is just that exit of the windows shell (powershell or command prompt)
|
||||
is not detected properly.
|
||||
</p>
|
||||
|
||||
<pre>
|
||||
curl http@secure@://@host@/docs/wsproxy > wsproxy
|
||||
chmod 755 wsproxy
|
||||
<h2>Local clients: using ssh proxy command </h2>
|
||||
|
||||
<p><code>wsproxy</code> is a command that can be used as a proxy command for SSH which performs the connection to the
|
||||
remote server. This command needs to be downloaded only once (see <a href="#downloads">downloads</a> below). It does not depend on
|
||||
the converge implementation but only on the websocket standards. Other tools that
|
||||
provide a mapping of stdio to a websocket can also be used instead of wsproxy.
|
||||
</p>
|
||||
<p>
|
||||
Next step is to run a local SSH or SFTP client:
|
||||
</p>
|
||||
|
||||
<pre>
|
||||
ssh -oServerAliveInterval=10 -oProxyCommand="wsproxy ws{{.secure}}://{{.host}}/client/ID" abc@localhost
|
||||
sftp -oServerAliveInterval=10 -oProxyCommand="wsproxy ws{{.secure}}://{{.host}}/client/ID" abc@localhost
|
||||
</pre>
|
||||
|
||||
<p>This is a command that can be used as a proxy command for SSH which performs the connection to the remote
|
||||
server.</p>
|
||||
<p>
|
||||
<code>abc</code> is a fixed user defined by converge. It has a very exciting password.
|
||||
</p>
|
||||
|
||||
<p>
|
||||
Next step is to run a local SSH or SFTP client:
|
||||
</p>
|
||||
<h2>Local clients: with a local TCP forwarding proxy</h2>
|
||||
|
||||
<pre>
|
||||
ssh -oServerAliveInterval=10 -oProxyCommand="wsproxy ws@secure@://@host@/client/ID" abc@localhost
|
||||
sftp -oServerAliveInterval=10 -oProxyCommand="wsproxy ws@secure@://@host@/client/ID" abc@localhost
|
||||
</pre>
|
||||
<p>
|
||||
This option is less convenient than the proxy command because it requires two separate
|
||||
commands to execute.
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<code>abc</code> is a fixed user defined by converge. It has a very exciting password.
|
||||
</p>
|
||||
<p>
|
||||
Local clients can connect using regular ssh and sftp commands through a tunnel that
|
||||
translates a local TCP port to a websocket connection in converge. See
|
||||
the <a href="#downloads">downloads</a> section.
|
||||
This runs a local client that allows SSH to port 10000 and connects to converge using
|
||||
a websocket connection.
|
||||
</p>
|
||||
|
||||
<h2>Local clients: with a local TCP forwarding proxy</h2>
|
||||
|
||||
This option is less convenient than the proxy command because it requires two separate
|
||||
commands to execute.
|
||||
<p>
|
||||
Next step is to run a local SSH of SFTP client:
|
||||
</p>
|
||||
|
||||
<p>
|
||||
Local clients can connect using regular ssh and sftp commands through a tunnel that
|
||||
translates a local TCP port to a websocket connection in converge.
|
||||
|
||||
First step is to download the tcptows program (see below):
|
||||
</p>
|
||||
|
||||
<pre>
|
||||
curl http@secure@://@host@/docs/tcptows > tcptows
|
||||
chmod 755 tcptows
|
||||
./tcptows 10000 ws@secure@://@host@/client/ID
|
||||
</pre>
|
||||
|
||||
<p>
|
||||
This runs a local client that allows SSH to port 10000 and connects to converge using
|
||||
a websocket connection.</p>
|
||||
|
||||
<p>
|
||||
Next step is to run a local SSH of SFTP client:
|
||||
</p>
|
||||
|
||||
<pre>
|
||||
<pre>
|
||||
ssh -oServerAliveInterval=10 -p 10000 abc@localhost
|
||||
sftp -oServerAliveInterval=10 -oPort=10000 abc@localhost
|
||||
</pre>
|
||||
|
||||
<p>
|
||||
<code>abc</code> is a fixed user defined by converge. It has a very exciting password.
|
||||
</p>
|
||||
<p>
|
||||
<code>abc</code> is a fixed user defined by converge. It has a very exciting password.
|
||||
</p>
|
||||
|
||||
<h1>Downloads</h1>
|
||||
<h1 id="downloads">Downloads</h1>
|
||||
|
||||
<ul>
|
||||
<li><a href="/docs/agent">agent</a>: The agent to run inside aa CI job
|
||||
</li>
|
||||
<li><a href="/docs/tcptows">tcptows</a>: TCP to WS tunnel for allowing regular
|
||||
SSH and SFTP clients to connect to converge.
|
||||
</li>
|
||||
<li><a href="/docs/wsproxy">wsproxy</a>: SSH proxy command that can be directly used by ssh
|
||||
</li>
|
||||
</ul>
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Component</th>
|
||||
<th>Purpose</th>
|
||||
<th>Linux</th>
|
||||
<th>Windows</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tr>
|
||||
<td>agent</td>
|
||||
<td>The agent to run inside aa CI job</td>
|
||||
<td><a href="/docs/agent">agent</a></td>
|
||||
<td><a href="/docs/agent.exe">agent.exe</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>wsproxy</td>
|
||||
<td>SSH proxy command that can be directly used by ssh</td>
|
||||
<td><a href="/docs/wsproxy">wsproxy</a></td>
|
||||
<td><a href="/docs/wsproxy.exe">wsproxy.exe</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>tcptows</td>
|
||||
<td>TCP to WS tunnel for allowing regular
|
||||
SSH and SFTP clients to connect to converge</td>
|
||||
<td><a href="/docs/tcptows">tcptows</a></td>
|
||||
<td><a href="/docs/tcptows.exe">tcptows.exe</a></td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user