Hi All
My name is Bao Quach, I am technical Lead developer at Australia NSW Transport.
We have consolidate all log files from our 3000 Microservices insub directories of central location “/opt/scats/scats_output_log” on Linux Mint 19.2
For example
/opt/scats/scats_output_log
/opt/scats/scats_output_log/24001
/opt/scats/scats_output_log/24002
/opt/scats/scats_output_log/3289
Where 24001 or 24002 or 3289 are traffic site intersection
Each site could have more 100 files with different file name.
I wrote a class called GrayLogSendGelfOutput (Please the class below) to connect to Garylog’s REST API http://:9000/api to save log file content from my global location to Graylog’s Elasticseach.
I have the following questions:-
Do you need to provide userid and password when invoke Garylog’s REST API http://:9000/api ? If so, can you provide Java code example.
.
That is,why I have built a Java Microservice which take all the log files from - global directory import into Garylog’s Elasticsearch, using Garylog’s REST API.
We want a simple solution and easy to setup up, support dynamic log files and zero administration.
I get an error in class o.g.g.transport.GelfTcpTransport : Channel disconnected!
when using “gelf-tcp” protocol to log files contents via Graylog’s REST API
Can you please have a look at the class GrayLogSendGelfOutput and let me know what I am missing.
Thank you for your help
Regards, Bao
GrayLogSendGelfOutput class
@Slf4j
@Service
public class GrayLogSendGelfOutput {
private static final String GELT_TCP_PROTOCOL_TYPE = "gelf-tcp";
private static final String GELT_UDP_PROTOCOL_TYPE = "gelf-udp";
private GelfTransport transport;
private final CountDownLatch transportInitialized = new CountDownLatch(1);
private String graylogProtocolType = "gelf-tcp";
private String graylogHost;
private int graylogPort;
private int graylogClientQueuesize = 512;
private int graylogClientSendBufferSize = -1;
private int graylogClientConnectTimeout = 5000;
private int graylogClientReconnectDelay = 1000;
private boolean graylogClientTcpNoDelay = true;
private boolean graylogIsClientTls = false;
private boolean graylogIsClientTlsVerifyCert= true;
private String graylogClientTlsCertChainFile = "none";
@Autowired
GrayLogSendGelfOutput(@Value("${graylog.protocol.type}") final String graylogProtocolType,
@Value("${graylog.host}") final String graylogHost,
@Value("${graylog.port}") final int graylogPort,
@Value("${graylog.client.queue.size}") final int graylogClientQueuesize,
@Value("${graylog.client.send.buffer.size}") final int graylogClientSendBufferSize,
@Value("${graylog.client.tls.cert.chain.file}") final String graylogClientTlsCertChainFile
)
{
this.graylogProtocolType = graylogProtocolType;
this.graylogHost = graylogHost;
this.graylogPort = graylogPort;
this.graylogClientQueuesize = graylogClientQueuesize;
this.graylogClientSendBufferSize = graylogClientSendBufferSize;
}
public void doStart() {
final GelfConfiguration clientConfig = new GelfConfiguration(graylogHost, graylogPort);
if(GELT_UDP_PROTOCOL_TYPE.equals(graylogProtocolType) ) {
clientConfig
.transport(GelfTransports.UDP)
.queueSize(graylogClientQueuesize)
.sendBufferSize(graylogClientSendBufferSize);
} else if(GELT_TCP_PROTOCOL_TYPE.equals(graylogProtocolType) ) {
clientConfig
.transport(GelfTransports.TCP)
.queueSize(graylogClientQueuesize)
.connectTimeout(graylogClientConnectTimeout)
.reconnectDelay(graylogClientReconnectDelay)
.tcpNoDelay(graylogClientTcpNoDelay)
.sendBufferSize(graylogClientSendBufferSize);
if (graylogIsClientTls) {
clientConfig.enableTls();
clientConfig.tlsTrustCertChainFile(new File(graylogClientTlsCertChainFile));
if (graylogIsClientTlsVerifyCert) {
clientConfig.enableTlsCertVerification();
} else {
clientConfig.disableTlsCertVerification();
}
}
}
log.info("Starting GELF transport: {}", clientConfig);
this.transport = GelfTransports.create(clientConfig);
transportInitialized.countDown();
}
public void doStop() {
}
public void write(Message message) {
//Uninterruptibles.awaitUninterruptibly(transportInitialized);
log.info("Write message.getInput() = {}", message.getInput());
try {
final GelfMessageBuilder messageBuilder = new GelfMessageBuilder(message.getMessage(), message.getSource())
.timestamp(message.getTimestamp().getMillis() / 1000.0)
.additionalFields(message.getFields().asMap());
if (message.getLevel() != null) {
messageBuilder.level(GelfMessageLevel.valueOf(message.getLevel().toString()));
} else {
messageBuilder.level(null);
}
if( transport == null ){
doStart();
}
transport.send(messageBuilder.build());
} catch (InterruptedException e) {
log.error("Failed to send message to Graylog", e);
}
}
public void createAndSendMessages(File filePath, String messageString) {
if (messageString.isEmpty()) {
// skip completely empty messages, they contain no useful information
log.info("createAndSendMessages messageString.isEmpty() ");
return;
}
String source = graylogHost;
DateTime timestamp = DateTime.now(DateTimeZone.UTC);
Message.Level level = null;
String input = filePath.getName();
Set<String> outputs = new HashSet<>();
outputs.add(filePath.getAbsolutePath());
MessageFields fields = getSCATSApplicationMessageFields(filePath.getName());
final Message message = new Message(messageString, source, timestamp, level, input, outputs, fields);
log.info("createAndSendMessages message.getTimestamp() = {} ", message.getTimestamp());
message.getFields().put("source_file", filePath.getAbsolutePath());
log.info("createAndSendMessages message.getSource().length() = {} ", message.getSource().length());
write(message);
}
public void sendLogFilesContent(File filePath, String message) {
log.info("sendLogFilesContent filePath.getAbsoluteFile() = {}", filePath.getAbsoluteFile());
Set<String> outputs = new HashSet<>();
outputs.add(filePath.getAbsolutePath());
createAndSendMessages(filePath, message);
}
public MessageFields getSCATSApplicationMessageFields(String programName, int priority){
MessageFields messageFields = new MessageFields();
messageFields.put("program", programName);
messageFields.put("priority", priority);
return messageFields;
}
public MessageFields getSCATSApplicationMessageFields(String programName){
return getSCATSApplicationMessageFields(programName, 3);
}