Rsyslog streaming doesn't work

Rsyslog streaming doesn’t work

Hello, I try to send nxlog Gelf TCP > Rsyslog (for buffering ) > Graylog.

The messages that rsylog receives has a delimiter #000.
So they look like,
{“version”:“1.1”,"_Keywords":-9214364837600034816,"…“timestamp”:1630649932,“level”:6}#000{“version”:“1.1”,"_Keywords":-9214364837600034816,"…“timestamp”:1630649932,“level”:6}

To test to send it to Graylog I use netcat.
echo -n ‘{“version”:“1.1”,"_Keywords":-9214364837600034816,"…“timestamp”:1630649932,“level”:6}’ | nc 192.168.XXX.XXX XXXX
the message receives and is shown in Graylog.

Now I have tested it with two messages,
after changing the Input from Null frame delimiter to new line delimiter.
echo -n ‘{“version”:“1.1”,"_Keywords":-9214364837600034816,"…“timestamp”:1630649932,“level”:6}’\n’{“version”:“1.1”,"_Keywords":-9214364837600034817,"…“timestamp”:1630649933,“level”:6}’ | nc 192.168.XXX.XXX XXXX
echo -n ‘{“version”:“1.1”,"_Keywords":-9214364837600034816,"…“timestamp”:1630649932,“level”:6}\n{“version”:“1.1”,"_Keywords":-9214364837600034817,"…“timestamp”:1630649933,“level”:6}’ | nc 192.168.XXX.XXX XXXX

in both cases ‘\n’ or \n only the first message appears.

Did I get a wrong control characters?

I tried \n\r and ‘\n\r’ too with the same result.

Or is it not possible to stream/send more messages?

Thanks for help.

Sven

Hello,

For us to help you further we need some more information about your configurations made. I’m not sure what is going on besides using Netcat and half your message is come through. This could be multiple settings misconfigured. Did this work before? Have you tried not to use Rsyslog and see if you get messages ingested first? What other troubleshooting have to tried besides ‘\n’ or \n ?
Knowing how you configured your environment will help us, help you better.

This may help.

Hello, here are the system Information I forgot to send.

  • nxlog = nxlog-ce-2.11.2190
  • rsyslog = rsyslogd 8.2001.0
  • graylog = Graylog 4.1.3+9d79c05

So what I now get working so far
nxlog syslog > rsyslog to gelf > Graylog GELF_TCP

But this doesn’t split off the Message.

nxlog.conf

#NoFreeOnExit TRUE

define ROOT     C:\Program Files (x86)\nxlog
define CERTDIR  %ROOT%\cert
define CONFDIR  %ROOT%\conf
define LOGDIR   %ROOT%\data
define LOGFILE  %LOGDIR%\nxlog.log
LogFile %LOGFILE%

Moduledir %ROOT%\modules
CacheDir  %ROOT%\data
Pidfile   %ROOT%\data\nxlog.pid
SpoolDir  %ROOT%\data

######################################################
############## Extensions ############################
<Extension syslog>
    Module      xm_syslog
</Extension>
########## INPUTS ###########
<Input in>
    Module      im_msvistalog
</Input>
########################################

<Processor p_anco_01>
  Module pm_transformer
  Exec $Hostname = hostname();
  OutputFormat syslog_rfc5424
</Processor>

################# OUTPUTS ##############
<Output out>
	Module      om_tcp    
    Host        192.168.xxx.xx
    Port        xxxx
    Exec        $raw_event = replace($raw_event, "\r\n", " ");
	Exec        $raw_event = replace($raw_event, "\t", " ");
</Output>
#######################################
#################### ROUTE  ###########
<Route 1>
    Path in => p_anco_01 => out
</Route>

==================================

cat /etc/rsyslog.conf

# /etc/rsyslog.conf configuration file for rsyslog
#
# For more information install rsyslog-doc and see
# /usr/share/doc/rsyslog-doc/html/configuration/index.html
#
# Default logging rules can be found in /etc/rsyslog.d/50-default.conf

#################
#### MODULES ####
#################

#module(load="imuxsock") # provides support for local system logging
#module(load="immark")  # provides --MARK-- message capability

# provides UDP syslog reception
#module(load="imudp")
#input(type="imudp" port="514")

# provides TCP syslog reception
module(load="imtcp")
input(type="imtcp" port="xxxx")

# Erlaubte Sender Definieren

$AllowedSender TCP, 192.168.0.0/16

# Speicherort Syslogmeldungen

$template RemoteLogs, "/var/log/%FROMHOST-IP%/syslog_tcp05.log"
*.* ?RemoteLogs

# Gelf Template
# https://www.rsyslog.com/doc/master/tutorials/gelf_forwarding.html
template(name="gelf" type="list") {
    constant(value="{\"version\":\"1.1\",")
    constant(value="\"host\":\"")
    property(name="hostname")
    constant(value="\",\"short_message\":\"")
    property(name="msg" format="json")
    constant(value="\",\"timestamp\":")
    property(name="timegenerated" dateformat="unixtimestamp")
    constant(value=",\"level\":\"")
    property(name="syslogseverity")
    constant(value="\"}")
}
action(type="omfwd" Target="192.168.xxx.xx" port="xxxx" Protocol="tcp" template="gelf" TCP_FrameDelimiter="0")

###########################
#### GLOBAL DIRECTIVES ####
###########################

#
# Use traditional timestamp format.
# To enable high precision timestamps, comment out the following line.
#
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat

# Filter duplicated messages
$RepeatedMsgReduction on

#
# Set the default permissions for all log files.
#
$FileOwner syslog
$FileGroup adm
$FileCreateMode 0640
$DirCreateMode 0755
$Umask 0022
$PrivDropToUser syslog
$PrivDropToGroup syslog

#
# Where to place spool and state files
#
$WorkDirectory /var/spool/rsyslog

#
# Include all config files in /etc/rsyslog.d/
#
$IncludeConfig /etc/rsyslog.d/*.conf

==========================================================

What I want to do, is sent nxlog gelf to Rsyslog and forward it to Graylog so that the automatic parsing could happen.

nxlog.conf

#NoFreeOnExit TRUE

define ROOT     C:\Program Files (x86)\nxlog
define CERTDIR  %ROOT%\cert
define CONFDIR  %ROOT%\conf
define LOGDIR   %ROOT%\data
define LOGFILE  %LOGDIR%\nxlog.log
LogFile %LOGFILE%

Moduledir %ROOT%\modules
CacheDir  %ROOT%\data
Pidfile   %ROOT%\data\nxlog.pid
SpoolDir  %ROOT%\data

######################################################
############## Extensions ############################

<Extension _gelf>
    Module      xm_gelf
</Extension>
########## INPUTS ###########
<Input in>
    Module      im_msvistalog
</Input>
########################################
################# OUTPUTS ##############
<Output out>
    Module      om_tcp    
    Host        192.168.xxx.xxx
    Port        xxxxx
    OutputType  GELF_TCP 

</Output>
#######################################
#################### ROUTE  ###########
<Route 1>
    Path in => out
</Route>

=====================================================

rsyslog.conf

root@vsd-nc-dev:/etc# cat rsyslog.conf_sic2021_09_03
# /etc/rsyslog.conf configuration file for rsyslog
#
# For more information install rsyslog-doc and see
# /usr/share/doc/rsyslog-doc/html/configuration/index.html
#
# Default logging rules can be found in /etc/rsyslog.d/50-default.conf


#################
#### MODULES ####
#################

#module(load="imuxsock") # provides support for local system logging
#module(load="immark")  # provides --MARK-- message capability

# provides UDP syslog reception
#module(load="imudp")
#input(type="imudp" port="514")

# provides TCP syslog reception
module(load="imtcp")
input(type="imtcp" port="xxxx")

# Erlaubte Sender Definieren

$AllowedSender TCP, 192.168.0.0/16

# Speicherort Syslogmeldungen

$template RemoteLogs, "/var/log/%FROMHOST-IP%/gelf_tcp03.log"
*.* ?RemoteLogs

template(name="foobar" type="string" string="%syslogtag%%msg%") action(type="omfwd" Target="192.168.xxx.xxx" port="xxxx" Protocol="tcp" template="foobar" TCP_FrameDelimiter="0")


###########################
#### GLOBAL DIRECTIVES ####
###########################

#
# Use traditional timestamp format.
# To enable high precision timestamps, comment out the following line.
#
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat

# Filter duplicated messages
$RepeatedMsgReduction on

#
# Set the default permissions for all log files.
#
$FileOwner syslog
$FileGroup adm
$FileCreateMode 0640
$DirCreateMode 0755
$Umask 0022
$PrivDropToUser syslog
$PrivDropToGroup syslog

#
# Where to place spool and state files
#
$WorkDirectory /var/spool/rsyslog

#
# Include all config files in /etc/rsyslog.d/
#
$IncludeConfig /etc/rsyslog.d/*.conf

===================================================

Working so far.

  • NXLog messages are stored on the Rsyslog Server
  • Only the first Message is shown in Graylog not the following

My though are it’s the Delimiter that is used in the rsyslog stream to Graylog.

If I watch the stream with tcpdump it shows
on the working stream to graylog

on the not working stream to Graylog

Hello @SR_CSV

Thank you for the added information. This helped to understand what you doing.

If your only using Rsyslog for buffering you might want to check this out.

I personally would choose either Rsyslog or Nxlog to ship log/s directly to Graylog input. I have tried using Rsyslog but unfortunately I was running into problems most of the time, specially when I want to adjust configurations on Graylog.

Example:

I believe what you want can be done either with Nxlog_GELF/TCP
OR Graylog Sidecar

To give you a better understanding this is my setup. My Graylog input is configured with Gelf/tcp/tls and my shipper is Nxlog. Nxlog is shipping logs from a Linux box to Graylog. These settings can be configured/adjusted for Gelf_TCP if you don’t want to use TLS.

NXLOG-CONFIG

########################################
# Global directives                    #
########################################
#User nxlog
#Group nxlog

LogFile /var/log/nxlog/nxlog.log
LogLevel INFO

########################################
# Modules                              #
########################################
<Extension _syslog>
    Module      xm_syslog
</Extension>

<Extension _gelf>
    Module      xm_gelf
</Extension>

<Input messages>
    Module       im_file
    FILE         "/var/log/messages"
    SavePos       TRUE
    ReadFromLast  TRUE
    PollInterval  1
    Exec  $Message = $raw_event;
</Input>

<Input secure>
    Module       im_file
    FILE         "/var/log/secure"
    SavePos       TRUE
    ReadFromLast  TRUE
    PollInterval  1
    Exec  $Message = $raw_event;
</Input>

<Input audit>
   Module       im_file
   FILE         "/var/log/audit/audit.log"
   SavePos       TRUE
   ReadFromLast  TRUE
   PollInterval  1
   Exec  $Message = $raw_event;
</Input>
<Input nextcloud>
   Module       im_file
   FILE         "/mnt/nextcloud-data1/nextcloud.log"
   SavePos       TRUE
   ReadFromLast  TRUE
   PollInterval  1
   Exec  $Message = $raw_event;
</Input>

<Output out>
    Module      om_ssl ### for tcp use your om_tcp
    Host        graylog.domain.com
    Port        51411
    OutputType  GELF_TCP
    CertFile        /var/lib/nxlog/cert/graylog3-certificate.pem
    CertKeyFile     /var/lib/nxlog/cert/graylog3-key.pem
    CAFile          /var/lib/nxlog/cert/cert3.pem
    KeyPass         secret
    AllowUntrusted  true
    Exec $short_message = $raw_event;
    Exec $Hostname = hostname_fqdn();
</Output>



########################################
# Routes                               #
########################################
<Route>
Path messages => out
</Route>

<Route>
Path secure => out
</Route>

<Route>
Path audit => out
</Route>

<Route>
Path nextcloud => out
</Route>

Here is my Graylog Input,

Results

Over all I believe you can simplify your setup.
Hope that helps

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.