1
0
Fork 0
mirror of https://github.com/Eggbertx/gochan.git synced 2025-09-05 11:06:23 -07:00

Implement file fingerprint, filename, and checksum banning via filter conditions

This commit is contained in:
Eggbertx 2024-08-17 16:26:11 -07:00
parent 83bc642674
commit 35860a8a6d
9 changed files with 455 additions and 47 deletions

View file

@ -23,9 +23,8 @@ import (
func editPost(checkedPosts []int, editBtn string, doEdit string, writer http.ResponseWriter, request *http.Request) {
password := request.FormValue("password")
wantsJSON := serverutil.IsRequestingJSON(request)
errEv := gcutil.LogError(nil).
Str("IP", gcutil.GetRealIP(request))
defer errEv.Discard()
infoEv, errEv := gcutil.LogRequest(request)
defer gcutil.LogDiscard(infoEv, errEv)
if editBtn == "Edit post" {
var err error
@ -84,9 +83,8 @@ func editPost(checkedPosts []int, editBtn string, doEdit string, writer http.Res
if upload != nil {
data["upload"] = upload
}
buf := bytes.NewBufferString("")
err = serverutil.MinifyTemplate(gctemplates.PostEdit, data, buf, "text/html")
if err != nil {
var buf bytes.Buffer
if err = serverutil.MinifyTemplate(gctemplates.PostEdit, data, &buf, "text/html"); err != nil {
errEv.Err(err).Caller().
Msg("Error executing edit post template")
server.ServeError(writer, "Error executing edit post template: "+err.Error(), wantsJSON, nil)
@ -95,33 +93,36 @@ func editPost(checkedPosts []int, editBtn string, doEdit string, writer http.Res
writer.Write(buf.Bytes())
}
if doEdit == "post" || doEdit == "upload" {
postid, err := strconv.Atoi(request.FormValue("postid"))
postIDstr := request.FormValue("postid")
postid, err := strconv.Atoi(postIDstr)
if err != nil {
errEv.Err(err).Caller().
Str("postid", request.FormValue("postid")).
Str("postid", postIDstr).
Msg("Invalid form data")
server.ServeError(writer, "Invalid form data: "+err.Error(), wantsJSON, map[string]interface{}{
"postid": postid,
})
return
}
gcutil.LogInt("postID", postid, infoEv, errEv)
post, err := gcsql.GetPostFromID(postid, true)
if err != nil {
errEv.Err(err).Caller().
Int("postid", postid).
Msg("Unable to find post")
server.ServeError(writer, "Unable to find post: "+err.Error(), wantsJSON, map[string]interface{}{
errEv.Err(err).Caller().Msg("Unable to find post")
server.ServeError(writer, "Unable to find post", wantsJSON, map[string]interface{}{
"postid": postid,
})
return
}
boardid, err := strconv.Atoi(request.FormValue("boardid"))
boardIDstr := request.FormValue("boardid")
boardid, err := strconv.Atoi(boardIDstr)
if err != nil {
errEv.Err(err).Caller().
Str("boardID", boardIDstr).
Msg("Invalid form data")
server.ServeError(writer, "Invalid form data: "+err.Error(), wantsJSON, nil)
return
}
gcutil.LogInt("boardID", boardid, infoEv, errEv)
rank := manage.GetStaffRank(request)
password := request.PostFormValue("password")
@ -148,7 +149,7 @@ func editPost(checkedPosts []int, editBtn string, doEdit string, writer http.Res
return
}
upload, err := uploads.AttachUploadFromRequest(request, writer, post, board)
upload, err := uploads.AttachUploadFromRequest(request, writer, post, board, gcutil.LogInfo(), errEv)
if err != nil {
server.ServeError(writer, err.Error(), wantsJSON, nil)
return
@ -220,7 +221,6 @@ func editPost(checkedPosts []int, editBtn string, doEdit string, writer http.Res
request.FormValue("editmsg"),
); err != nil {
errEv.Err(err).Caller().
Int("postid", post.ID).
Msg("Unable to edit post")
server.ServeError(writer, "Unable to edit post: "+err.Error(), wantsJSON, map[string]interface{}{
"postid": post.ID,
@ -236,6 +236,7 @@ func editPost(checkedPosts []int, editBtn string, doEdit string, writer http.Res
server.ServeErrorPage(writer, "Error rebuilding front page: "+err.Error())
}
http.Redirect(writer, request, post.WebPath(), http.StatusFound)
infoEv.Msg("Post edited")
return
}
}

222
pkg/gcsql/filterhandlers.go Normal file
View file

@ -0,0 +1,222 @@
package gcsql
import (
"errors"
"fmt"
"net/http"
"regexp"
"strings"
)
const (
StringField FieldType = iota
BooleanField
)
var (
// filterFieldHandlers = make(map[string]FilterConditionHandler)
filterFieldHandlers map[string]FilterConditionHandler
)
type FieldType int
type ConditionMatchFunc func(*http.Request, *Post, *Upload, *FilterCondition) (bool, error)
type conditionHandler struct {
fieldType FieldType
matchFunc ConditionMatchFunc
}
func (ch *conditionHandler) Type() FieldType {
return ch.fieldType
}
func (ch *conditionHandler) CheckMatch(request *http.Request, post *Post, upload *Upload, fc *FilterCondition) (bool, error) {
return ch.matchFunc(request, post, upload, fc)
}
// FilterConditionHandler handles filter conditions, providing support for checking a field
type FilterConditionHandler interface {
Type() FieldType
CheckMatch(*http.Request, *Post, *Upload, *FilterCondition) (bool, error)
}
func validateConditionHandler(field string, matchFunc ConditionMatchFunc) error {
if _, ok := filterFieldHandlers[field]; ok {
return fmt.Errorf("field %q is already registered", field)
} else if field == "" {
return errors.New("condition field must not be empty")
} else if matchFunc == nil {
return errors.New("condition match function must not be nil")
}
return nil
}
func RegisterStringConditionHandler(field string, matchFunc ConditionMatchFunc) error {
if err := validateConditionHandler(field, matchFunc); err != nil {
return err
}
filterFieldHandlers[field] = &conditionHandler{
fieldType: StringField,
matchFunc: matchFunc,
}
return nil
}
func RegisterBooleanConditionHandler(field string, matchFunc ConditionMatchFunc) error {
if err := validateConditionHandler(field, matchFunc); err != nil {
return err
}
filterFieldHandlers[field] = &conditionHandler{
fieldType: BooleanField,
matchFunc: matchFunc,
}
return nil
}
func firstPost(post *Post, global bool) (bool, error) {
var board int
var err error
if !global {
board, err = post.GetBoardID()
if err != nil {
return false, err
}
}
query := `SELECT COUNT(*) FROM DBPREFIXposts `
params := []any{post.IP}
if board > 0 {
query += ` LEFT JOIN DBPREFIXthreads ON thread_id = DBPREFIXthreads.id WHERE ip = ? AND board_id = ?`
params = append(params, board)
} else {
query += ` WHERE ip = PARAM_ATON`
}
var count int
err = QueryRowTimeoutSQL(nil, query, params, []any{&count})
return count > 0, err
}
func matchString(fc *FilterCondition, checkStr string) (bool, error) {
if fc.IsRegex {
re, err := regexp.Compile(fc.Search)
if err != nil {
return false, err
}
return re.MatchString(checkStr), nil
}
return strings.Contains(checkStr, fc.Search), nil
}
func init() {
filterFieldHandlers = map[string]FilterConditionHandler{
"name": &conditionHandler{
fieldType: StringField,
matchFunc: func(r *http.Request, p *Post, _ *Upload, fc *FilterCondition) (bool, error) {
return matchString(fc, p.Name)
},
},
"trip": &conditionHandler{
fieldType: StringField,
matchFunc: func(r *http.Request, p *Post, _ *Upload, fc *FilterCondition) (bool, error) {
return matchString(fc, p.Name)
},
},
"email": &conditionHandler{
fieldType: StringField,
matchFunc: func(r *http.Request, p *Post, _ *Upload, fc *FilterCondition) (bool, error) {
return matchString(fc, p.Email)
},
},
"subject": &conditionHandler{
fieldType: StringField,
matchFunc: func(r *http.Request, p *Post, _ *Upload, fc *FilterCondition) (bool, error) {
return matchString(fc, p.Subject)
},
},
"body": &conditionHandler{
fieldType: StringField,
matchFunc: func(r *http.Request, p *Post, _ *Upload, fc *FilterCondition) (bool, error) {
return matchString(fc, p.MessageRaw)
},
},
"firsttimeboard": &conditionHandler{
fieldType: BooleanField,
matchFunc: func(r *http.Request, p *Post, _ *Upload, _ *FilterCondition) (bool, error) {
return firstPost(p, false)
},
},
"notfirsttimeboard": &conditionHandler{
fieldType: BooleanField,
matchFunc: func(r *http.Request, p *Post, _ *Upload, _ *FilterCondition) (bool, error) {
first, err := firstPost(p, false)
return !first, err
},
},
"firsttimesite": &conditionHandler{
fieldType: BooleanField,
matchFunc: func(r *http.Request, p *Post, _ *Upload, _ *FilterCondition) (bool, error) {
return firstPost(p, true)
},
},
"notfirsttimesite": &conditionHandler{
fieldType: BooleanField,
matchFunc: func(r *http.Request, p *Post, _ *Upload, _ *FilterCondition) (bool, error) {
first, err := firstPost(p, true)
return !first, err
},
},
"isop": &conditionHandler{
fieldType: BooleanField,
matchFunc: func(r *http.Request, p *Post, _ *Upload, _ *FilterCondition) (bool, error) {
return p.IsTopPost, nil
},
},
"notop": &conditionHandler{
fieldType: BooleanField,
matchFunc: func(r *http.Request, p *Post, _ *Upload, _ *FilterCondition) (bool, error) {
return !p.IsTopPost, nil
},
},
"hasfile": &conditionHandler{
fieldType: BooleanField,
matchFunc: func(r *http.Request, p *Post, u *Upload, fc *FilterCondition) (bool, error) {
return u != nil, nil
},
},
"nofile": &conditionHandler{
fieldType: BooleanField,
matchFunc: func(r *http.Request, p *Post, u *Upload, fc *FilterCondition) (bool, error) {
return u == nil, nil
},
},
"filename": &conditionHandler{
fieldType: StringField,
matchFunc: func(r *http.Request, p *Post, u *Upload, fc *FilterCondition) (bool, error) {
if u == nil {
return false, nil
}
return matchString(fc, u.OriginalFilename)
},
},
"checksum": &conditionHandler{
fieldType: StringField,
matchFunc: func(r *http.Request, p *Post, u *Upload, fc *FilterCondition) (bool, error) {
if u == nil {
return false, nil
}
return u.Checksum == fc.Search, nil
},
},
// TODO: register file-related checks in the uploads package so they can be handled before the file is saved and to avoid potential
// cyclical dependencies
// "ahash": &conditionHandler{
// fieldType: StringField,
// }
"useragent": &conditionHandler{
fieldType: StringField,
matchFunc: func(r *http.Request, p *Post, _ *Upload, fc *FilterCondition) (bool, error) {
return matchString(fc, r.UserAgent())
},
},
}
}

View file

@ -3,9 +3,13 @@ package gcsql
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"net/http"
"time"
"github.com/rs/zerolog"
)
const (
@ -19,29 +23,14 @@ const (
)
var (
ErrInvalidConditionField = errors.New("unrecognized conditional field")
ErrInvalidMatchAction = errors.New("unrecognized filter action")
ErrInvalidConditionField = errors.New("unrecognized filter condition field")
ErrInvalidMatchAction = errors.New("unrecognized filter match action")
ErrInvalidFilter = errors.New("unrecognized filter id")
ErrNoConditions = errors.New("error has no match conditions")
)
type ActiveFilter int
type wordFilterFilter int
// whereClause returns part of the where clause of a SQL string. If and is true, it starts with AND, otherwise it starts with WHERE
func (af ActiveFilter) whereClause(and bool) string {
out := " WHERE "
if and {
out = " AND "
}
if af == OnlyActiveFilters {
return out + "is_active = TRUE"
} else if af == OnlyInactiveFilters {
return out + "is_active = FALSE"
}
return ""
}
// GetFilterByID returns the filter with the given ID, and an error if one occured
func GetFilterByID(id int) (*Filter, error) {
var filter Filter
@ -129,7 +118,6 @@ func getFiltersByBoardDir(dir string, includeAllBoards bool, show ActiveFilter,
); err != nil {
return nil, err
}
fmt.Println(filter.ID, filter.MatchDetail, filter.StaffNote)
filters = append(filters, filter)
}
return filters, rows.Close()
@ -342,6 +330,123 @@ func (f *Filter) SetBoardIDs(ids ...int) error {
return tx.Commit()
}
type matchHitJSON struct {
Post *Post
Upload *Upload
MatchConditions []string
UserAgent string
}
// handleMatch takes the set action after the filter has been found to match the given post. It returns any errors that occured
func (f *Filter) handleMatch(post *Post, request *http.Request) error {
var conditionFields []string
for _, condition := range f.conditions {
// it's assumed that f.Condition() was already called and returned no errors so we don't need to check it again
conditionFields = append(conditionFields, condition.Field)
}
upload, err := post.GetUpload()
if err != nil {
return err
}
ba, err := json.Marshal(matchHitJSON{Post: post, Upload: upload, MatchConditions: conditionFields, UserAgent: request.UserAgent()})
if err != nil {
return err
}
if _, err = ExecTimeoutSQL(nil, `INSERT INTO DBPREFIXfilter_hits(filter_id,post_data) VALUES(?,?)`, f.ID, string(ba)); err != nil {
return err
}
switch f.MatchAction {
case "reject":
return nil
case "ban":
return NewIPBan(&IPBan{
IPBanBase: IPBanBase{
IsActive: true,
StaffID: *f.StaffID,
Permanent: true,
CanAppeal: true,
AppealAt: time.Now(),
StaffNote: fmt.Sprintf("banned by filter #%d", f.ID),
Message: f.MatchDetail,
},
RangeStart: post.IP,
RangeEnd: post.IP,
IssuedAt: time.Now(),
})
case "log":
// already logged
return nil
}
return ErrInvalidMatchAction
}
// checkIfMatch checks the filter's conditions to see if it matches the post and handles it according to the MatchAction
// value, returning true if it matched and false otherwise
func (f *Filter) checkIfMatch(post *Post, upload *Upload, request *http.Request, errEv *zerolog.Event) (bool, error) {
conditions, err := f.Conditions()
if err != nil {
return false, err
}
match := true
for _, condition := range conditions {
if !match {
break
}
if match, err = condition.testCondition(post, upload, request, errEv); err != nil {
return false, err
}
}
if match {
}
return match, nil
}
func (fc *FilterCondition) testCondition(post *Post, upload *Upload, request *http.Request, errEv *zerolog.Event) (bool, error) {
handler, ok := filterFieldHandlers[fc.Field]
if !ok {
return false, ErrInvalidConditionField
}
match, err := handler.CheckMatch(request, post, upload, fc)
if err != nil {
errEv.Err(err).Caller().
Str("field", fc.Field).
Int("filterID", fc.FilterID).
Int("filterConditionID", fc.ID).Send()
err = errors.New("unable to check filter condition")
}
return match, err
}
// DoPostFiltering checks the filters against the given post. If a match is found, its respective action is taken and the filter
// is returned. It logs any errors it receives and returns a sanitized error (if one occured) that can be shown to the end user
func DoPostFiltering(post *Post, upload *Upload, boardID int, request *http.Request, errEv *zerolog.Event) (*Filter, error) {
filters, err := GetFiltersByBoardID(post.ID, true, OnlyActiveFilters)
if err != nil {
errEv.Err(err).Caller().Msg("Unable to get filter list")
return nil, errors.New("unable to get post filter list")
}
var match bool
for f, filter := range filters {
if match, err = filter.checkIfMatch(post, upload, request, errEv); err != nil {
errEv.Err(err).Caller().
Int("filterID", filter.ID).
Msg("Unable to check filter for a match")
return nil, errors.New("unable to check filter for a match")
}
if match {
filter.handleMatch(post, request)
return &filters[f], nil
}
}
return nil, nil
}
// SetFilterActive updates the filter with the given id, setting its active status and returning an error if one occured
func SetFilterActive(id int, active bool) error {
_, err := ExecTimeoutSQL(nil, `UPDATE DBPREFIXfilters SET is_active = ? WHERE id = ?`, active, id)

View file

@ -1,6 +1,8 @@
package initsql
import (
"net/http"
"path"
"strconv"
"text/template"
@ -9,6 +11,7 @@ import (
"github.com/gochan-org/gochan/pkg/gcsql"
"github.com/gochan-org/gochan/pkg/gctemplates"
"github.com/gochan-org/gochan/pkg/gcutil"
"github.com/gochan-org/gochan/pkg/posting/uploads"
)
func banMaskTmplFunc(ban gcsql.IPBan) string {
@ -131,4 +134,22 @@ func init() {
"getBoardDefaultStyle": getBoardDefaultStyleTmplFunc,
"sectionBoards": sectionBoardsTmplFunc,
})
gcsql.RegisterStringConditionHandler("ahash", func(r *http.Request, p *gcsql.Post, u *gcsql.Upload, fc *gcsql.FilterCondition) (bool, error) {
if u == nil {
return false, nil
}
boardID, err := strconv.Atoi(r.PostFormValue("boardid"))
if err != nil {
// boardid is assumed to have already been checked, but just in case...
return false, err
}
dir, err := gcsql.GetBoardDir(boardID)
if err != nil {
return false, err
}
fingerprint, err := uploads.GetFileFingerprint(path.Join(
config.GetSystemCriticalConfig().DocumentRoot,
dir, "src", u.Filename))
return fingerprint == fc.Search, err
})
}

View file

@ -20,6 +20,23 @@ var (
ErrNotConnected = errors.New("error connecting to database")
)
// ActiveFilter is used for optionally limiting the results of tables with an is_active column to
type ActiveFilter int
// whereClause returns part of the where clause of a SQL string. If and is true, it starts with AND, otherwise it starts with WHERE
func (af ActiveFilter) whereClause(and bool) string {
out := " WHERE "
if and {
out = " AND "
}
if af == OnlyActiveFilters {
return out + "is_active = TRUE"
} else if af == OnlyInactiveFilters {
return out + "is_active = FALSE"
}
return ""
}
// BeginTx begins a new transaction for the gochan database. It uses a background context
func BeginTx() (*sql.Tx, error) {
return BeginContextTx(context.Background())

View file

@ -30,8 +30,10 @@ var (
{Value: "email", Text: "Email"},
{Value: "subject", Text: "Subject"},
{Value: "body", Text: "Message body"},
{Value: "firsttime", Text: "First time poster"},
{Value: "notfirsttime", Text: "Not a first time poster"},
{Value: "firsttimeboard", Text: "First time poster (board)"},
{Value: "notfirsttimeboard", Text: "Not a first time poster (board)"},
{Value: "firsttimesite", Text: "First time poster (site-wide)"},
{Value: "notfirsttimesite", Text: "Not a first time poster (site-wide)"},
{Value: "isop", Text: "Is OP"},
{Value: "notop", Text: "Is reply"},
{Value: "hasfile", Text: "Has file"},

View file

@ -43,7 +43,7 @@ func showBanpage(ban *gcsql.IPBan, post *gcsql.Post, postBoard *gcsql.Board, wri
Msg("Rejected post from banned IP")
}
// checks the post for spam. It returns true if a ban page or an error page was served (causing MakePost() to return)
// checks the post IP against the IP range ban list. It returns true if a ban page or an error page was served (causing MakePost() to return)
func checkIpBan(post *gcsql.Post, postBoard *gcsql.Board, writer http.ResponseWriter, request *http.Request) bool {
ipBan, err := gcsql.CheckIPBan(post.IP, postBoard.ID)
if err != nil {
@ -62,7 +62,7 @@ func checkIpBan(post *gcsql.Post, postBoard *gcsql.Board, writer http.ResponseWr
return true
}
func checkUsernameBan(post *gcsql.Post, postBoard *gcsql.Board, writer http.ResponseWriter, request *http.Request) bool {
/* func checkUsernameBan(post *gcsql.Post, postBoard *gcsql.Board, writer http.ResponseWriter, request *http.Request) bool {
nameTrip := post.Name
if post.Tripcode != "" {
nameTrip += "!" + post.Tripcode
@ -93,7 +93,7 @@ func checkUsernameBan(post *gcsql.Post, postBoard *gcsql.Board, writer http.Resp
Bool("banIsRegex", nameBan.IsRegex).
Msg("Rejected post with banned name/tripcode")
return true
}
} */
func handleAppeal(writer http.ResponseWriter, request *http.Request, infoEv *zerolog.Event, errEv *zerolog.Event) {
banIDstr := request.FormValue("banid")

View file

@ -285,12 +285,33 @@ func MakePost(writer http.ResponseWriter, request *http.Request) {
return
}
// filters, err := gcsql.GetFiltersByBoardID(post.ID, true, gcsql.OnlyActiveFilters)
// if err != nil {
// errEv.Err(err).Caller().Msg("Unable to get filter list")
// server.ServeError(writer, "Unable to get post filter list", wantsJSON, nil)
// return
// }
// var match bool
// for _, filter := range filters {
// if match, err = filter.CheckIfMatch(post, request); err != nil {
// errEv.Err(err).Caller().
// Int("filterID", filter.ID).
// Msg("Unable to check filter for a match")
// server.ServeError(writer, "Unable to check post filters", wantsJSON, nil)
// return
// }
// if match {
// }
// }
if checkIpBan(post, postBoard, writer, request) {
return
}
if checkUsernameBan(post, postBoard, writer, request) {
return
}
// if checkUsernameBan(post, postBoard, writer, request) {
// return
// }
captchaSuccess, err := submitCaptchaResponse(request)
if err != nil {
@ -324,7 +345,7 @@ func MakePost(writer http.ResponseWriter, request *http.Request) {
return
}
upload, err := uploads.AttachUploadFromRequest(request, writer, post, postBoard)
upload, err := uploads.AttachUploadFromRequest(request, writer, post, postBoard, infoEv, errEv)
documentRoot := config.GetSystemCriticalConfig().DocumentRoot
var filePath, thumbPath, catalogThumbPath string
if upload != nil {
@ -353,6 +374,29 @@ func MakePost(writer http.ResponseWriter, request *http.Request) {
return
}
filter, err := gcsql.DoPostFiltering(post, upload, boardID, request, errEv)
if err != nil {
server.ServeError(writer, err.Error(), wantsJSON, nil)
return
}
if filter != nil {
infoEv.Int("filterID", filter.ID).Msg("Found a matching filter")
os.Remove(filePath)
os.Remove(thumbPath)
os.Remove(catalogThumbPath)
switch filter.MatchAction {
case "reject":
rejectReason := filter.MatchDetail
if rejectReason == "" {
rejectReason = "Post rejected"
}
server.ServeError(writer, rejectReason, wantsJSON, nil)
case "ban":
checkIpBan(post, postBoard, writer, request)
}
return
}
if err = post.Insert(emailCommand != "sage", postBoard.ID, false, false, false, false); err != nil {
errEv.Err(err).Caller().
Str("sql", "postInsertion").

View file

@ -74,11 +74,7 @@ func init() {
// AttachUploadFromRequest reads an incoming HTTP request and processes any incoming files.
// It returns the upload (if there was one) and whether or not any errors were served (meaning
// that it should stop processing the post
func AttachUploadFromRequest(request *http.Request, writer http.ResponseWriter, post *gcsql.Post, postBoard *gcsql.Board) (*gcsql.Upload, error) {
infoEv, errEv := gcutil.LogRequest(request)
defer func() {
gcutil.LogDiscard(infoEv, errEv)
}()
func AttachUploadFromRequest(request *http.Request, writer http.ResponseWriter, post *gcsql.Post, postBoard *gcsql.Board, infoEv *zerolog.Event, errEv *zerolog.Event) (*gcsql.Upload, error) {
file, handler, err := request.FormFile("imagefile")
if errors.Is(err, http.ErrMissingFile) {
// no file was submitted with the form