diff --git a/codes/deploy/main.sh b/codes/deploy/main.sh index d719b79..6bfdb14 100644 --- a/codes/deploy/main.sh +++ b/codes/deploy/main.sh @@ -1,12 +1,13 @@ #!/bin/bash ################################################################################### -# 环境部署脚本 +# Linux Centos 环境部署脚本 +# Author: Zhang Peng ################################################################################### function printBeginning() { cat << EOF *********************************************************************************** -* Welcome to using the deployment script for linux. +* Welcome to using the deployment script for Centos. * Author: Zhang Peng *********************************************************************************** EOF @@ -44,13 +45,16 @@ cat << EOF =================================== Deploy Menu =================================== 【1 - System Environment】 - [1-2 | libs] install commonly libs. + [sys] initial system environment + [libs] install commonly libs 【2 - Common Tools】 [2 | tools] install all tools. - [2-1 | git] install git. [2-2 | jdk8] install jdk8. - [2-3 | maven] install maven. [2-4 | nginx] install nginx. - [2-5 | nodejs] install node.js. [2-6 | tomcat] install tomcat8. + [git] install git [svn] install svn + [jdk8] install jdk8 [jdk8] install jdk8 + [maven] install maven [nginx] install nginx + [nodejs] install node.js [tomcat] install tomcat8 + [elk] install elk Press to exit Please input key： @@ -63,15 +67,18 @@ function chooseOper() { while read key do case ${key} in - 1-2 | libs) ${filepath}/lib/install-libs.sh;; + sys) ${filepath}/sys/init.sh;; + libs) ${filepath}/lib/install-libs.sh;; 2 | tools) ${filepath}/tool/install-all.sh;; - 2-1 | git) ${filepath}/tool/git/install-git.sh;; - 2-2 | jdk8) ${filepath}/tool/jdk/install-jdk8.sh;; - 2-3 | maven) ${filepath}/tool/maven/install-maven.sh;; - 2-4 | nginx) ${filepath}/tool/nginx/install-nginx.sh;; - 2-5 | nodejs) ${filepath}/tool/nodejs/install-nodejs.sh;; - 2-6 | tomcat) ${filepath}/tool/tomcat/install-tomcat8.sh;; + git) ${filepath}/tool/git/install-git.sh;; + svn) ${filepath}/tool/git/install-svn.sh;; + jdk8) ${filepath}/tool/jdk/install-jdk8.sh;; + maven) ${filepath}/tool/maven/install-maven.sh;; + nginx) ${filepath}/tool/nginx/install-nginx.sh;; + nodejs) ${filepath}/tool/nodejs/install-nodejs.sh;; + tomcat) ${filepath}/tool/tomcat/install-tomcat8.sh;; + elk) ${filepath}/tool/elk/install-elk.sh;; * ) echo "invalid key";; esac diff --git a/codes/deploy/sys/init.sh b/codes/deploy/sys/init.sh new file mode 100644 index 0000000..6fbbdb2 --- /dev/null +++ b/codes/deploy/sys/init.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +################################################################################### +# Linux Centos 环境初始化脚本（设置环境配置、安装基本的命令工具） +# Author: Zhang Peng +################################################################################### + +cat << EOF +*********************************************************************************** +* The initialization of linux environment is begin. +*********************************************************************************** +EOF + +filepath=$(cd "$(dirname "$0")"; pwd) + +# 设置环境配置，不了解具体修改内容的情况下，请勿执行 +# ./set-config.sh + +# 安装命令行工具 +${filepath}/install-cmd-tool.sh + +# 时钟同步工具 +yum -y install ntp +# 同步上海交通大学网络中心NTP服务器 +echo "* 4 * * * /usr/sbin/ntpdate ntp.sjtu.edu.cn > /dev/null 2>&1" >> /var/spool/cron/root +# 以一个服务器时间为标准定时更新时间（有时需要以公司中的服务器作为标准） +#echo "*/30 * * * * /usr/local/bin/ntpdate 192.168.16.182" >> /var/spool/cron/root + +cat << EOF +*********************************************************************************** +* The initialization of linux environment is over. +*********************************************************************************** +EOF diff --git a/codes/deploy/sys/install-cmd-tool.sh b/codes/deploy/sys/install-cmd-tool.sh new file mode 100644 index 0000000..d6139c6 --- /dev/null +++ b/codes/deploy/sys/install-cmd-tool.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +################################################################################### +# 安装基本的命令工具 +# Author: Zhang Peng +# +# 如果不知道某个命令工具是由哪个包提供的，使用 yum provides xxx +# 或 yum whatprovides xxx 来查找 +################################################################################### + +################################################################################### +# 执行本脚本后支持的命令工具 +# 核心工具：df、du、chkconfig +# 网络工具：ifconfig、netstat、route +# IP工具：ip、ss、ping、tracepath、traceroute +# DNS工具：dig、host、nslookup、whois +# 端口工具：lsof、nc、telnet +# 下载工具：curl、wget +# 防火墙工具：iptables +# 编辑工具：emacs、vim +# 流量工具：iftop、nethogs +# 抓包工具：tcpdump +################################################################################### +# 核心工具 +echo -e "\n>>>>>>>>> install coreutils(df、du)" +yum install -y coreutils +echo -e "\n>>>>>>>>> install chkconfig" +yum install -y chkconfig + +# 网络工具 +echo -e "\n>>>>>>>>> install net-tools(ifconfig、netstat、route)" +yum install -y net-tools + +# IP工具 +echo -e "\n>>>>>>>>> install iputils(ping、tracepath)" +yum install -y iputils +echo -e "\n>>>>>>>>> install traceroute" +yum install -y traceroute +echo -e "\n>>>>>>>>> install iproute(ip、ss)" +yum install -y iproute + +# 端口工具 +echo -e "\n>>>>>>>>> install lsof" +yum install -y lsof +echo -e "\n>>>>>>>>> install nc" +yum install -y nc +echo -e "\n>>>>>>>>> install netstat" +yum install -y netstat + +# DNS工具 +echo -e "\n>>>>>>>>> install bind-utils(dig、host、nslookup)" +yum install -y bind-utils +echo -e "\n>>>>>>>>> install whois" +yum install -y whois + +# 下载工具 +echo -e "\n>>>>>>>>> install curl" +yum install -y curl +echo -e "\n>>>>>>>>> install wget" +yum install -y wget + +# 防火墙工具 +echo -e "\n>>>>>>>>> install iptables" +yum install -y iptables + +# 编辑工具 +echo -e "\n>>>>>>>>> install emacs" +yum install -y emacs +echo -e "\n>>>>>>>>> install vim" +yum install -y vim + +# 流量工具 +echo -e "\n>>>>>>>>> install iftop" +yum install -y iftop +echo -e "\n>>>>>>>>> install nethogs" +yum install -y nethogs + +# 抓包工具 +echo -e "\n>>>>>>>>> install tcpdump" +yum install -y tcpdump diff --git a/codes/deploy/sys/set-config.sh b/codes/deploy/sys/set-config.sh new file mode 100644 index 0000000..f27e753 --- /dev/null +++ b/codes/deploy/sys/set-config.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +################################################################################### +# Linux Centos 设置环境配置脚本 +# Author: Zhang Peng +# +# 不了解脚本中配置意图的情况下，不要贸然执行此脚本 +################################################################################### + +# 获取当前机器 IP +ip="" +getDeviceIp() { + ip=`ifconfig eth0 | grep "inet addr" | awk '{ print $2}' | awk -F: '{print $2}'` + if [ "$ip" == "" ] + then + ip=`ifconfig ens32 | grep "inet"|grep "broadcast" | awk '{ print $2}' | awk -F: '{print $1}'` + fi + + if [ "$ip" == "" ] + then + ip=`echo $1` + fi + + if [ "${ip}" == "" ] + then + echo "无法获取IP地址" + exit 0 + fi +} + +host=`hostname` + +#set the file limit +cat >> /etc/security/limits.conf << EOF + * - nofile 65535 + * - nproc 65535 +EOF + +#set system language utf8 +cat > /etc/sysconfig/i18n << EOF +LANG="zh_CN.UTF-8" +EOF + +#set DNS +cat >> /etc/hosts << EOF +${ip} ${host} +EOF + +#set the control-alt-delete to guard against the miSUSE +sed -i 's#exec /sbin/shutdown -r now#\#exec /sbin/shutdown -r now#' /etc/init/control-alt-delete.conf + +#disable selinux +sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config + +#Full multiuser mode +sed -i 's/id:5:initdefault:/id:3:initdefault:/' /etc/inittab + +#tune kernel parametres +cat >> /etc/sysctl.conf << EOF +net.ipv4.tcp_tw_reuse = 1 +net.ipv4.tcp_tw_recycle = 1 +net.ipv4.tcp_fin_timeout = 2 +net.ipv4.tcp_syncookies = 1 +net.ipv4.tcp_keepalive_time = 1200 +net.ipv4.tcp_max_syn_backlog = 16384 +net.core.netdev_max_backlog = 16384 +net.core.somaxconn = 32768 +net.core.wmem_default = 8388608 +net.core.rmem_default = 8388608 +net.core.rmem_max = 16777216 +net.core.wmem_max = 16777216 +net.ipv4.tcp_timestamps = 0 +net.ipv4.route.gc_timeout = 100 +net.ipv4.tcp_synack_retries = 1 +net.ipv4.tcp_syn_retries = 1 +net.ipv4.tcp_mem = 94500000 915000000 927000000 +net.ipv4.tcp_max_orphans = 3276800 +net.ipv4.ip_local_port_range = 2000 65535 +net.ipv4.tcp_max_tw_buckets = 5000 +vm.swappiness=10 +EOF + +#disable the ipv6 +cat > /etc/modprobe.d/ipv6.conf << EOF +alias net-pf-10 off +options ipv6 disable=1 +EOF +echo "NETWORKING_IPV6=off" >> /etc/sysconfig/network diff --git a/codes/deploy/tool/elk/config/filebeat.yml b/codes/deploy/tool/elk/config/filebeat.yml new file mode 100644 index 0000000..c11956c --- /dev/null +++ b/codes/deploy/tool/elk/config/filebeat.yml @@ -0,0 +1,177 @@ +###################### Filebeat Configuration Example ######################### + +# This file is an example configuration file highlighting only the most common +# options. The filebeat.reference.yml file from the same directory contains all the +# supported options with more comments. You can use it as a reference. +# +# You can find the full configuration reference here: +# https://www.elastic.co/guide/en/beats/filebeat/index.html + +# For more available modules and options, please see the filebeat.reference.yml sample +# configuration file. + +#=========================== Filebeat prospectors ============================= + +filebeat.prospectors: + +# Each - is a prospector. Most options can be set at the prospector level, so +# you can use different prospectors for various configurations. +# Below are the prospector specific configurations. + +- type: log + + # Change to true to enable this prospector configuration. + enabled: true + + # Paths that should be crawled and fetched. Glob based paths. + paths: + #- /var/log/*.log + #- c:\programdata\elasticsearch\logs\* + - /home/zp/log/*.log + + # Exclude lines. A list of regular expressions to match. It drops the lines that are + # matching any regular expression from the list. + #exclude_lines: ['^DBG'] + + # Include lines. A list of regular expressions to match. It exports the lines that are + # matching any regular expression from the list. + #include_lines: ['^ERR', '^WARN'] + + # Exclude files. A list of regular expressions to match. Filebeat drops the files that + # are matching any regular expression from the list. By default, no files are dropped. + #exclude_files: ['.gz$'] + + # Optional additional fields. These fields can be freely picked + # to add additional information to the crawled log files for filtering + #fields: + # level: debug + # review: 1 + + ### Multiline options + + # Mutiline can be used for log messages spanning multiple lines. This is common + # for Java Stack Traces or C-Line Continuation + + # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [ + #multiline.pattern: ^\[ + + # Defines if the pattern set under pattern should be negated or not. Default is false. + #multiline.negate: false + + # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern + # that was (not) matched before or after or as long as a pattern is not matched based on negate. + # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash + #multiline.match: after + + +#============================= Filebeat modules =============================== + +filebeat.config.modules: + # Glob pattern for configuration loading + path: ${path.config}/modules.d/*.yml + + # Set to true to enable config reloading + reload.enabled: true + + # Period on which files under path should be checked for changes + #reload.period: 10s + +#==================== Elasticsearch template setting ========================== + +setup.template.settings: + index.number_of_shards: 3 + #index.codec: best_compression + #_source.enabled: false + +#================================ General ===================================== + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +name: 127.0.0.1 + +# The tags of the shipper are included in their own field with each +# transaction published. +#tags: ["service-X", "web-tier"] + +# Optional fields that you can specify to add additional information to the +# output. +fields: + profile: development + + +#============================== Dashboards ===================================== +# These settings control loading the sample dashboards to the Kibana index. Loading +# the dashboards is disabled by default and can be enabled either by setting the +# options here, or by using the `-setup` CLI flag or the `setup` command. +setup.dashboards.enabled: true + +# The URL from where to download the dashboards archive. By default this URL +# has a value which is computed based on the Beat name and version. For released +# versions, this URL points to the dashboard archive on the artifacts.elastic.co +# website. +#setup.dashboards.url: + +#============================== Kibana ===================================== + +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# This requires a Kibana endpoint configuration. +setup.kibana: + + # Kibana Host + # Scheme and port can be left out and will be set to the default (http and 5601) + # In case you specify and additional path, the scheme is required: http://localhost:5601/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 + host: "192.168.28.11:5601" + +#============================= Elastic Cloud ================================== + +# These settings simplify using filebeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + +#================================ Outputs ===================================== + +# Configure what output to use when sending the data collected by the beat. + +#-------------------------- Elasticsearch output ------------------------------ +#output.elasticsearch: + # Array of hosts to connect to. + #hosts: ["192.168.28.11:9200"] + + # Optional protocol and basic auth credentials. + protocol: "http" + #username: "elastic" + #password: "changeme" + +#----------------------------- Logstash output -------------------------------- +output.logstash: + # The Logstash hosts + hosts: ["192.168.28.32:5044"] + + # Optional SSL. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + +#================================ Logging ===================================== + +# Sets log level. The default log level is info. +# Available log levels are: critical, error, warning, info, debug +#logging.level: debug + +# At debug level, you can selectively enable logging only for some components. +# To enable all selectors use ["*"]. Examples of other selectors are "beat", +# "publish", "service". +logging.selectors: ["*"] diff --git a/codes/deploy/tool/elk/config/logback.xml b/codes/deploy/tool/elk/config/logback.xml new file mode 100644 index 0000000..64c96bd --- /dev/null +++ b/codes/deploy/tool/elk/config/logback.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + %d{HH:mm:ss.SSS} [%thread] [%-5p] %c{36}.%M - %m%n + + + + + + + + ${user.dir}/logs/${FILE_NAME}-all.%d{yyyy-MM-dd}.log + 30 + + + + + 30MB + + + + %d{HH:mm:ss.SSS} [%thread] [%-5p] %c{36}.%M - %m%n + + + + + 192.168.28.32:9251 + + {"appname":"javatool"} + + + + + + + + + + + + + + + + + + diff --git a/codes/deploy/tool/elk/config/logstash-input-tcp.conf b/codes/deploy/tool/elk/config/logstash-input-tcp.conf new file mode 100644 index 0000000..8bd897e --- /dev/null +++ b/codes/deploy/tool/elk/config/logstash-input-tcp.conf @@ -0,0 +1,12 @@ +input { + tcp { + port => 9251 + codec => json_lines + mode => server + tags => ["javaapp"] + } +} +output { + elasticsearch { hosts => ["localhost:9200"] } + stdout { codec => rubydebug } +} diff --git a/codes/deploy/tool/elk/install-elk.sh b/codes/deploy/tool/elk/install-elk.sh new file mode 100644 index 0000000..3ce8523 --- /dev/null +++ b/codes/deploy/tool/elk/install-elk.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash + +# 本脚本为一键式安装 ELK 脚本 +# 执行脚本前，请先执行以下命令，创建用户 +# groupadd elk +# useradd -g elk elk +# passwd elk + +# 获取当前设备IP +IP="" +getDeviceIp() { + IP=`ifconfig eth0 | grep "inet" | awk '{ print $2}' | awk -F: '{print $2}'` + if [ "$IP" == "" ]; then + IP=`ifconfig eth0 | grep "inet" | awk '{ print $2}'` + fi + if [ "$IP" == "" ]; then + IP=`ifconfig ens32 | grep "inet"|grep "broadcast" | awk '{ print $2}' | awk -F: '{print $1}'` + fi + + if [ "${IP}" == "" ]; then + echo " " + echo " 请输入服务器IP地址................ " + echo " " + exit 0 + else + echo "当前设备IP: $IP" + fi +} + +# 检查文件是否存在，不存在则退出脚本 +checkFileExist() { + if [ ! -f "$1" ] + then + echo "关键文件 $1 找不到，脚本执行结束" + exit 0 + fi +} + +init() { + mkdir -p ${ELASTIC_SOFTWARE_PATH} + getDeviceIp +} + +# 安装 elasticsearch +installElasticsearch() { + cd ${ELASTIC_SOFTWARE_PATH} + wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${version}.tar.gz + tar -xzf elasticsearch-${version}.tar.gz +} + +installRuby() { + cd ${RUBY_SOFTWARE_PATH} + wget https://cache.ruby-lang.org/pub/ruby/2.5/ruby-2.5.0.tar.gz + tar -xzf ruby-2.5.0.tar.gz + cd ruby-2.5.0 + ./configure + make & make install +} + +# 安装 logstash +installLogstash() { + cd ${ELASTIC_SOFTWARE_PATH} + wget https://artifacts.elastic.co/downloads/logstash/logstash-${version}.tar.gz + tar -xzf logstash-${version}.tar.gz +} + +# 安装 kibana +installKibana() { + cd ${ELASTIC_SOFTWARE_PATH} + wget https://artifacts.elastic.co/downloads/kibana/kibana-${version}-linux-x86_64.tar.gz + tar -xzf kibana-${version}-linux-x86_64.tar.gz +} + +# 安装 filebeat +installFilebeat() { + cd ${ELASTIC_SOFTWARE_PATH} + wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-${version}-linux-x86_64.tar.gz + tar -zxf filebeat-${version}-linux-x86_64.tar.gz +} + +# 替换 Elasticsearch 配置 +# 1. 替换 192.168.0.1 为本机 IP +replaceElasticsearchConfig() { + cp ${ELASTIC_SOFTWARE_PATH}/elasticsearch-${version}/config/elasticsearch.yml ${ELASTIC_SOFTWARE_PATH}/elasticsearch-${version}/config/elasticsearch.yml.bak + sed -i "s/#network.host: 192.168.0.1/network.host: ${IP}/g" ${ELASTIC_SOFTWARE_PATH}/elasticsearch-${version}/config/elasticsearch.yml + touch ${ELASTIC_SOFTWARE_PATH}/elasticsearch-${version}/bin/nohup.out +} + +replaceLogstashConfig() { + cp ${ELASTIC_SOFTWARE_PATH}/logstash-${version}/config/logstash.yml ${ELASTIC_SOFTWARE_PATH}/logstash-${version}/config/logstash.yml.bak + sed -i "s/# http.host: \"127.0.0.1\"/ http.host: ${IP}/g" ${ELASTIC_SOFTWARE_PATH}/logstash-${version}/config/logstash.yml + touch ${ELASTIC_SOFTWARE_PATH}/logstash-${version}/bin/nohup.out + cd ${ELASTIC_SOFTWARE_PATH}/logstash-${version}/bin + wget https://github.com/dunwu/linux-notes/blob/master/codes/deploy/tool/elk/config/logstash-input-tcp.conf +} + +# 替换 Kibana 配置 +# 1. 替换 localhost 为本机 IP +replaceKibanaConfig() { + cp ${ELASTIC_SOFTWARE_PATH}/kibana-${version}-linux-x86_64/config/kibana.yml ${ELASTIC_SOFTWARE_PATH}/kibana-${version}-linux-x86_64/config/kibana.yml.bak + sed -i "s/#server.host: \"localhost\"/server.host: ${IP}/g" ${ELASTIC_SOFTWARE_PATH}/kibana-${version}-linux-x86_64/config/kibana.yml + sed -i "s/#elasticsearch.url: \"http://localhost:9200\"/#elasticsearch.url: \"${IP}\"/g" ${ELASTIC_SOFTWARE_PATH}/kibana-${version}-linux-x86_64/config/kibana.yml + touch ${ELASTIC_SOFTWARE_PATH}/kibana-${version}-linux-x86_64/bin/nohup.out +} + +# 替换 Filebeat 配置 +replaceFilebeatConfig() { + cp ${ELASTIC_SOFTWARE_PATH}/filebeat-${version}-linux-x86_64/filebeat.yml ${ELASTIC_SOFTWARE_PATH}/filebeat-${version}-linux-x86_64/filebeat.yml.bak + cd ${ELASTIC_SOFTWARE_PATH}/filebeat-${version}-linux-x86_64 + wget https://github.com/dunwu/linux-notes/blob/master/codes/deploy/tool/elk/config/filebeat.yml + sed -i 's/127.0.0.1/'"${IP}"'/g' ${ELASTIC_SOFTWARE_PATH}/filebeat-${version}-linux-x86_64/filebeat.yml +} + +# 为 elk.elk 用户设置权限 +setPrivilegeForUser() { + chown -R elk.elk ${ELASTIC_SOFTWARE_PATH} + chown -R elk.elk /var/log/ +} +######################################## MAIN ######################################## +echo -e "\n>>>>>>>>> install elk" + +version=6.1.1 +RUBY_SOFTWARE_PATH=/opt/software/ruby +ELASTIC_SOFTWARE_PATH=/opt/software/elastic +ELASTIC_SETTINGS_PATH=/opt/software/elastic/settings + +init + +installElasticsearch +replaceElasticsearchConfig + +installLogstash +replaceLogstashConfig + +installKibana +replaceKibanaConfig + +installFilebeat +replaceFilebeatConfig + +#setPrivilegeForUser + diff --git a/codes/deploy/tool/svn/install-svn.sh b/codes/deploy/tool/svn/install-svn.sh new file mode 100644 index 0000000..aa04e60 --- /dev/null +++ b/codes/deploy/tool/svn/install-svn.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +echo -e "\n>>>>>>>>> install subversion" + +yum install -y subversion