Import with nxlog and collector sidecar

Graylog Add-ons
, , ,
1

Hello,

I have problem with manual log import to graylog with collector sidecar.

I need to import 3-5gb data on weekly base to graylog, I get raw/plain logs in multiple folders ziped and when put it in read folder of sidecar collector he send data to graylog but not all data because he send as GELF. How to setup a nxlog to send plain/raw data and then parse in graylog.
On live import I send plain/raw data and then parse in graylog and work like a charm.

My nxlog conf.

define ROOT /usr/bin

<Extension gelf>
  Module xm_gelf
</Extension>
<Extension 5a26a74cc194d60889e14bb9-multiline>
  Module xm_multiline
  HeaderLine /^-./
</Extension>
<Extension 5a27b096c194d60443c4626f-multiline>
  Module xm_multiline
  HeaderLine /^\d{4}-\d{2}-\d{2}/
</Extension>

<Processor 5a26a74cc194d60889e14bb9-buffer>
  Module pm_buffer
  MaxSize 16384
  Type Mem
</Processor>
<Processor 5a27b096c194d60443c4626f-buffer>
  Module pm_buffer
  MaxSize 16384
  Type Mem
</Processor>


User nxlog
Group nxlog

Moduledir /usr/lib/nxlog/modules
CacheDir /var/spool/collector-sidecar/nxlog
PidFile /var/run/graylog/collector-sidecar/nxlog.pid


define LOGFILE /var/log/graylog/collector-sidecar/nxlog.log
LogFile %LOGFILE%
LogLevel INFO

<Extension logrotate>
    Module  xm_fileop
    <Schedule>
        When    @daily
        Exec    file_cycle('%LOGFILE%', 7);
     </Schedule>
</Extension>




<Input 5a26a74cc194d60889e14bb9>
        Module im_file
        File '/swisslog/*.log*'
        PollInterval 30
        SavePos True
        ReadFromLast True
        Recursive True
        RenameCheck True
        Exec $FileName = file_name(); # Send file name with each message
        InputType 5a26a74cc194d60889e14bb9-multiline
</Input>
<Input 5a27b096c194d60443c4626f>
        Module im_file
        File '/swisslog/*.txt*'
        PollInterval 6
        SavePos True
        ReadFromLast True
        Recursive True
        RenameCheck True
        Exec $FileName = file_name(); # Send file name with each message
        InputType 5a27b096c194d60443c4626f-multiline
</Input>





<Output 5a26a719c194d60889e14b81>
        Module om_tcp
        Host 172.16.11.110
        Port 5045
        OutputType  GELF_TCP
        Exec $short_message = $raw_event; # Avoids truncation of the short_message field.
        Exec $gl2_source_collector = 'f2ee341e-78b5-458d-bc30-38b32b81d85b';
        Exec $collector_node_id = 'graylog-collector-sidecar';
        Exec $Hostname = hostname_fqdn();
</Output>
<Output 5a27b070c194d60443c46243>
        Module om_tcp
        Host 172.16.11.110
        Port 5055
        OutputType  GELF_TCP
        Exec $short_message = $raw_event; # Avoids truncation of the short_message field.
        Exec $gl2_source_collector = 'f2ee341e-78b5-458d-bc30-38b32b81d85b';
        Exec $collector_node_id = 'graylog-collector-sidecar';
        Exec $Hostname = hostname_fqdn();
</Output>


<Route route-0>
  Path 5a26a74cc194d60889e14bb9 => 5a26a74cc194d60889e14bb9-buffer => 5a26a719c194d60889e14b81
</Route>
<Route route-1>
  Path 5a27b096c194d60443c4626f => 5a27b096c194d60443c4626f-buffer => 5a27b070c194d60443c46243
</Route>

and my parser/pipeline stage 0 on graylog server

rule "itc5_extract_log"
when
   has_field("message")
then
    let message_field = to_string($message.message);
    // grok expression
    
    // Note: the expression must both match nornal syslog on the ITC and the syslog of the ITC logger
    
    // "%{TIMESTAMP_ISO8601:timestamp} %{WORD:hostname;string} #%{SEQNUM:seqnum;int} %{WORD:facility;string} %{LOGLEVEL:loglevel;string} %{SYSLOGPROG:prog}: (\{%{BASE10NUM:millis}\} )?([\[](?<category>[^\]]*)\] [(](?<traceid>[^)]*)[)])?\s?%{GREEDYDATA:msg;string}
    // example: let gr = grok(pattern, value, [only_named_captures]) : GrokMatch$GrokResult
    
    
    
    
    let parsed_fields = grok(pattern: "%{TIMESTAMP_ISO8601:logtime} %{WORD:hostname;string} #%{SEQNUM:seqnum;int} %{WORD:facility;string} %{LOGLEVEL:loglevel;string} %{SYSLOGPROG:prog}: (\\{%{BASE10NUM:millis}\\} )?([\\[](?<category>[^\\]]*)\\] [(](?<traceid>[^)]*)[)])?\\s?%{GREEDYDATA:msg;string}", value: message_field);
    set_fields(parsed_fields);

    // set_field("debug", to_string(parsed_fields));
end

Please can someone help me with that!

2

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.