add flow support in template (i.e javascript scripting) (#4015)

* add flow logic

* progress

* working POC

* fix string slice normalization issue in variables

* update

* fix nil panic

* remove poll()

* load file with sandbox and more

* fix failing integration tests

* JS: log: print in vardump format

* fix missing id in protocols

* fix proto prefix in template context

* flow: add unit tests

* conditional flow support using flow

* fix proto callbacks + more unit tests

* adds integration test

* conditional flow: check if req has any matchers

* fix lint error

* deprecate iterate-all+ missing multi-proto implementation

* fix ip input in raw request

* JS: feat dedupe object+ more builtin funcs

* feat: hide protocol result using hide

* feat: async execution

* complete async execution support

* fix condition-flow without any matchers

* refactor: template executer package (tmplexec)

* flow executor working

* fix data race in templateCtx

* templateCtx redesign

* fix failing unit test

* add multiprotocol support to deprecated syntax

* fix race condition in utils & tlsx

* add documentation in flow package

* remove regions.txt file

* fix minor issue with self contained templates

* fix typos of copilot

* dep + misc update

* fix reqID: use req.Type instead of template.Type

---------

Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
This commit is contained in:
Tarun Koyalwar
2023-08-31 18:03:01 +05:30
committed by GitHub
parent 8125b6805c
commit f7fe99f806
78 changed files with 2577 additions and 640 deletions

View File

@@ -32,6 +32,8 @@ type Request struct {
operators.Operators `yaml:",inline,omitempty"`
CompiledOperators *operators.Operators `yaml:"-"`
// ID is the optional id of the request
ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the dns request,description=ID is the optional ID of the DNS Request"`
// description: |
// Engine type
Engine []string `yaml:"engine,omitempty" jsonschema:"title=engine,description=Engine,enum=python,enum=powershell,enum=command"`
@@ -96,7 +98,7 @@ func (request *Request) Requests() int {
// GetID returns the ID for the request if any.
func (request *Request) GetID() string {
return ""
return request.ID
}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
@@ -115,10 +117,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
// inject all template context values as gozero env variables
variables := protocolutils.GenerateVariables(input.MetaInput.Input, false, nil)
// add template context values
variables = generators.MergeMaps(variables, request.options.GetTemplateCtx(input.MetaInput).GetAll())
// optionvars are vars passed from CLI or env variables
optionVars := generators.BuildPayloadFromOptions(request.options.Options)
variablesMap := request.options.Variables.Evaluate(variables)
variables = generators.MergeMaps(variablesMap, variables, optionVars)
variables = generators.MergeMaps(variablesMap, variables, optionVars, request.options.Constants)
for name, value := range variables {
v := fmt.Sprint(value)
v, interactshURLs = request.options.Interactsh.Replace(v, interactshURLs)
@@ -151,6 +155,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
data["template-id"] = request.options.TemplateID
data["template-info"] = request.options.TemplateInfo
// expose response variables in proto_var format
// this is no-op if the template is not a multi protocol template
request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, data)
// add variables from template context before matching/extraction
data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll())
if request.options.Interactsh != nil {
request.options.Interactsh.MakePlaceholders(interactshURLs, data)
}