leandrof/linux
1
0
Fork 0
forked from Minki/linux
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tty: Replace ASYNC_NORMAL_ACTIVE bit and update atomically

Browse Source 
Replace ASYNC_NORMAL_ACTIVE bit in the tty_port::flags field with
TTY_PORT_ACTIVE bit in the tty_port::iflags field. Introduce helpers
tty_port_set_active() and tty_port_active() to abstract atomic bit ops.

Extract state changes from port lock sections, as this usage is
broken and confused; the state transitions are protected by the
tty lock (which mutually excludes parallel open/close/hangup),
and no user tests the active state while holding the port lock.

Signed-off-by: Peter Hurley <peter@hurleysoftware.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
This commit is contained in:
Peter Hurley 2016-04-09 17:53:22 -07:00 committed by Greg Kroah-Hartman
parent 5604a98e2f
commit 807c8d81f4
11 changed files with 53 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
drivers/isdn/i4l/isdn_tty.c
View File

@ -1622,7 +1622,7 @@ isdn_tty_hangup(struct tty_struct *tty)
return; return;
isdn_tty_shutdown(info); isdn_tty_shutdown(info);
port->count = 0; port->count = 0;
port->flags &= ~ASYNC_NORMAL_ACTIVE; tty_port_set_active(port, 0);
port->tty = NULL; port->tty = NULL;
wake_up_interruptible(&port->open_wait); wake_up_interruptible(&port->open_wait);
} }
@ -1979,7 +1979,7 @@ isdn_tty_find_icall(int di, int ch, setup_parm *setup)
#endif #endif
if ( if (
#ifndef FIX_FILE_TRANSFER #ifndef FIX_FILE_TRANSFER
(info->port.flags & ASYNC_NORMAL_ACTIVE) && tty_port_active(&info->port) &&
#endif #endif
(info->isdn_driver == -1) && (info->isdn_driver == -1) &&
(info->isdn_channel == -1) && (info->isdn_channel == -1) &&
@ -2018,8 +2018,6 @@ isdn_tty_find_icall(int di, int ch, setup_parm *setup)
return (wret == 2) ? 3 : 0; return (wret == 2) ? 3 : 0;
} }
#define TTY_IS_ACTIVE(info) (info->port.flags & ASYNC_NORMAL_ACTIVE)
int int
isdn_tty_stat_callback(int i, isdn_ctrl *c) isdn_tty_stat_callback(int i, isdn_ctrl *c)
{ {
@ -2077,7 +2075,7 @@ isdn_tty_stat_callback(int i, isdn_ctrl *c)
#ifdef ISDN_TTY_STAT_DEBUG #ifdef ISDN_TTY_STAT_DEBUG
printk(KERN_DEBUG "tty_STAT_DCONN ttyI%d\n", info->line); printk(KERN_DEBUG "tty_STAT_DCONN ttyI%d\n", info->line);
#endif #endif
if (TTY_IS_ACTIVE(info)) { if (tty_port_active(&info->port)) {
if (info->dialing == 1) { if (info->dialing == 1) {
info->dialing = 2; info->dialing = 2;
return 1; return 1;
@ -2088,7 +2086,7 @@ isdn_tty_stat_callback(int i, isdn_ctrl *c)
#ifdef ISDN_TTY_STAT_DEBUG #ifdef ISDN_TTY_STAT_DEBUG
printk(KERN_DEBUG "tty_STAT_DHUP ttyI%d\n", info->line); printk(KERN_DEBUG "tty_STAT_DHUP ttyI%d\n", info->line);
#endif #endif
if (TTY_IS_ACTIVE(info)) { if (tty_port_active(&info->port)) {
if (info->dialing == 1) if (info->dialing == 1)
isdn_tty_modem_result(RESULT_BUSY, info); isdn_tty_modem_result(RESULT_BUSY, info);
if (info->dialing > 1) if (info->dialing > 1)
@ -2118,7 +2116,7 @@ isdn_tty_stat_callback(int i, isdn_ctrl *c)
* waiting for it and * waiting for it and
* set DCD-bit of its modem-status. * set DCD-bit of its modem-status.
*/ */
if (TTY_IS_ACTIVE(info) || if (tty_port_active(&info->port) ||
(info->port.blocked_open && (info->port.blocked_open &&
(info->emu.mdmreg[REG_DCD] & BIT_DCD))) { (info->emu.mdmreg[REG_DCD] & BIT_DCD))) {
info->msr |= UART_MSR_DCD; info->msr |= UART_MSR_DCD;
@ -2145,7 +2143,7 @@ isdn_tty_stat_callback(int i, isdn_ctrl *c)
#ifdef ISDN_TTY_STAT_DEBUG #ifdef ISDN_TTY_STAT_DEBUG
printk(KERN_DEBUG "tty_STAT_BHUP ttyI%d\n", info->line); printk(KERN_DEBUG "tty_STAT_BHUP ttyI%d\n", info->line);
#endif #endif
if (TTY_IS_ACTIVE(info)) { if (tty_port_active(&info->port)) {
#ifdef ISDN_DEBUG_MODEM_HUP #ifdef ISDN_DEBUG_MODEM_HUP
printk(KERN_DEBUG "Mhup in ISDN_STAT_BHUP\n"); printk(KERN_DEBUG "Mhup in ISDN_STAT_BHUP\n");
#endif #endif
@ -2157,7 +2155,7 @@ isdn_tty_stat_callback(int i, isdn_ctrl *c)
#ifdef ISDN_TTY_STAT_DEBUG #ifdef ISDN_TTY_STAT_DEBUG
printk(KERN_DEBUG "tty_STAT_NODCH ttyI%d\n", info->line); printk(KERN_DEBUG "tty_STAT_NODCH ttyI%d\n", info->line);
#endif #endif
if (TTY_IS_ACTIVE(info)) { if (tty_port_active(&info->port)) {
if (info->dialing) { if (info->dialing) {
info->dialing = 0; info->dialing = 0;
info->last_l2 = -1; info->last_l2 = -1;
@ -2183,14 +2181,14 @@ isdn_tty_stat_callback(int i, isdn_ctrl *c)
return 1; return 1;
#ifdef CONFIG_ISDN_TTY_FAX #ifdef CONFIG_ISDN_TTY_FAX
case ISDN_STAT_FAXIND: case ISDN_STAT_FAXIND:
if (TTY_IS_ACTIVE(info)) { if (tty_port_active(&info->port)) {
isdn_tty_fax_command(info, c); isdn_tty_fax_command(info, c);
} }
break; break;
#endif #endif
#ifdef CONFIG_ISDN_AUDIO #ifdef CONFIG_ISDN_AUDIO
case ISDN_STAT_AUDIO: case ISDN_STAT_AUDIO:
if (TTY_IS_ACTIVE(info)) { if (tty_port_active(&info->port)) {
switch (c->parm.num[0]) { switch (c->parm.num[0]) {
case ISDN_AUDIO_DTMF: case ISDN_AUDIO_DTMF:
if (info->vonline) { if (info->vonline) {

2
drivers/tty/amiserial.c
View File

@ -1493,7 +1493,7 @@ static void rs_hangup(struct tty_struct *tty)
rs_flush_buffer(tty); rs_flush_buffer(tty);
shutdown(tty, info); shutdown(tty, info);
info->tport.count = 0; info->tport.count = 0;
info->tport.flags &= ~ASYNC_NORMAL_ACTIVE; tty_port_set_active(&info->tport, 0);
info->tport.tty = NULL; info->tport.tty = NULL;
wake_up_interruptible(&info->tport.open_wait); wake_up_interruptible(&info->tport.open_wait);
} }

5
drivers/tty/rocket.c
View File

@ -1042,9 +1042,10 @@ static void rp_close(struct tty_struct *tty, struct file *filp)
} }
} }
spin_lock_irq(&port->lock); spin_lock_irq(&port->lock);
info->port.flags &= ~(ASYNC_INITIALIZED | ASYNC_NORMAL_ACTIVE); port->flags &= ~ASYNC_INITIALIZED;
tty->closing = 0; tty->closing = 0;
spin_unlock_irq(&port->lock); spin_unlock_irq(&port->lock);
tty_port_set_active(port, 0);
mutex_unlock(&port->mutex); mutex_unlock(&port->mutex);
tty_port_tty_set(port, NULL); tty_port_tty_set(port, NULL);
@ -1624,7 +1625,7 @@ static int rp_write(struct tty_struct *tty,
/* Write remaining data into the port's xmit_buf */ /* Write remaining data into the port's xmit_buf */
while (1) { while (1) {
/* Hung up ? */ /* Hung up ? */
if (!test_bit(ASYNCB_NORMAL_ACTIVE, &info->port.flags)) if (!tty_port_active(&info->port))
goto end; goto end;
c = min(count, XMIT_BUF_SIZE - info->xmit_cnt - 1); c = min(count, XMIT_BUF_SIZE - info->xmit_cnt - 1);
c = min(c, XMIT_BUF_SIZE - info->xmit_head); c = min(c, XMIT_BUF_SIZE - info->xmit_head);

8
drivers/tty/serial/crisv10.c
View File

@ -3648,8 +3648,8 @@ rs_close(struct tty_struct *tty, struct file * filp)
schedule_timeout_interruptible(info->port.close_delay); schedule_timeout_interruptible(info->port.close_delay);
wake_up_interruptible(&info->port.open_wait); wake_up_interruptible(&info->port.open_wait);
} }
info->port.flags &= ~ASYNC_NORMAL_ACTIVE;
local_irq_restore(flags); local_irq_restore(flags);
tty_port_set_active(&info->port, 0);
/* port closed */ /* port closed */
@ -3732,7 +3732,7 @@ rs_hangup(struct tty_struct *tty)
shutdown(info); shutdown(info);
info->event = 0; info->event = 0;
info->port.count = 0; info->port.count = 0;
info->port.flags &= ~ASYNC_NORMAL_ACTIVE; tty_port_set_active(&info->port, 0);
info->port.tty = NULL; info->port.tty = NULL;
wake_up_interruptible(&info->port.open_wait); wake_up_interruptible(&info->port.open_wait);
} }
@ -3756,7 +3756,7 @@ block_til_ready(struct tty_struct *tty, struct file * filp,
* then make the check up front and then exit. * then make the check up front and then exit.
*/ */
if ((filp->f_flags & O_NONBLOCK) || tty_io_error(tty)) { if ((filp->f_flags & O_NONBLOCK) || tty_io_error(tty)) {
info->port.flags |= ASYNC_NORMAL_ACTIVE; tty_port_set_active(&info->port, 1);
return 0; return 0;
} }
@ -3825,7 +3825,7 @@ block_til_ready(struct tty_struct *tty, struct file * filp,
#endif #endif
if (retval) if (retval)
return retval; return retval;
info->port.flags |= ASYNC_NORMAL_ACTIVE; tty_port_set_active(&info->port, 1);
return 0; return 0;
} }

8
drivers/tty/serial/serial_core.c
View File

@ -1418,12 +1418,12 @@ static void uart_close(struct tty_struct *tty, struct file *filp)
uart_change_pm(state, UART_PM_STATE_OFF); uart_change_pm(state, UART_PM_STATE_OFF);
spin_lock_irq(&port->lock); spin_lock_irq(&port->lock);
} }
spin_unlock_irq(&port->lock);
tty_port_set_active(port, 0);
/* /*
* Wake up anyone trying to open this port. * Wake up anyone trying to open this port.
*/ */
clear_bit(ASYNCB_NORMAL_ACTIVE, &port->flags);
spin_unlock_irq(&port->lock);
wake_up_interruptible(&port->open_wait); wake_up_interruptible(&port->open_wait);
mutex_unlock(&port->mutex); mutex_unlock(&port->mutex);
@ -1501,13 +1501,13 @@ static void uart_hangup(struct tty_struct *tty)
pr_debug("uart_hangup(%d)\n", tty->index); pr_debug("uart_hangup(%d)\n", tty->index);
mutex_lock(&port->mutex); mutex_lock(&port->mutex);
if (port->flags & ASYNC_NORMAL_ACTIVE) { if (tty_port_active(port)) {
uart_flush_buffer(tty); uart_flush_buffer(tty);
uart_shutdown(tty, state); uart_shutdown(tty, state);
spin_lock_irqsave(&port->lock, flags); spin_lock_irqsave(&port->lock, flags);
port->count = 0; port->count = 0;
clear_bit(ASYNCB_NORMAL_ACTIVE, &port->flags);
spin_unlock_irqrestore(&port->lock, flags); spin_unlock_irqrestore(&port->lock, flags);
tty_port_set_active(port, 0);
tty_port_tty_set(port, NULL); tty_port_tty_set(port, NULL);
if (!uart_console(state->uart_port)) if (!uart_console(state->uart_port))
uart_change_pm(state, UART_PM_STATE_OFF); uart_change_pm(state, UART_PM_STATE_OFF);

6
drivers/tty/synclink.c
View File

@ -3201,7 +3201,7 @@ static void mgsl_hangup(struct tty_struct *tty)
shutdown(info); shutdown(info);
info->port.count = 0; info->port.count = 0;
info->port.flags &= ~ASYNC_NORMAL_ACTIVE; tty_port_set_active(&info->port, 0);
info->port.tty = NULL; info->port.tty = NULL;
wake_up_interruptible(&info->port.open_wait); wake_up_interruptible(&info->port.open_wait);
@ -3269,7 +3269,7 @@ static int block_til_ready(struct tty_struct *tty, struct file * filp,
if (filp->f_flags & O_NONBLOCK || tty_io_error(tty)) { if (filp->f_flags & O_NONBLOCK || tty_io_error(tty)) {
/* nonblock mode is set or port is not enabled */ /* nonblock mode is set or port is not enabled */
port->flags |= ASYNC_NORMAL_ACTIVE; tty_port_set_active(port, 1);
return 0; return 0;
} }
@ -3338,7 +3338,7 @@ static int block_til_ready(struct tty_struct *tty, struct file * filp,
__FILE__,__LINE__, tty->driver->name, port->count ); __FILE__,__LINE__, tty->driver->name, port->count );
if (!retval) if (!retval)
port->flags |= ASYNC_NORMAL_ACTIVE; tty_port_set_active(port, 1);
return retval; return retval;

6
drivers/tty/synclink_gt.c
View File

@ -756,9 +756,9 @@ static void hangup(struct tty_struct *tty)
spin_lock_irqsave(&info->port.lock, flags); spin_lock_irqsave(&info->port.lock, flags);
info->port.count = 0; info->port.count = 0;
info->port.flags &= ~ASYNC_NORMAL_ACTIVE;
info->port.tty = NULL; info->port.tty = NULL;
spin_unlock_irqrestore(&info->port.lock, flags); spin_unlock_irqrestore(&info->port.lock, flags);
tty_port_set_active(&info->port, 0);
mutex_unlock(&info->port.mutex); mutex_unlock(&info->port.mutex);
wake_up_interruptible(&info->port.open_wait); wake_up_interruptible(&info->port.open_wait);
@ -3268,7 +3268,7 @@ static int block_til_ready(struct tty_struct *tty, struct file *filp,
if (filp->f_flags & O_NONBLOCK || tty_io_error(tty)) { if (filp->f_flags & O_NONBLOCK || tty_io_error(tty)) {
/* nonblock mode is set or port is not enabled */ /* nonblock mode is set or port is not enabled */
port->flags |= ASYNC_NORMAL_ACTIVE; tty_port_set_active(port, 1);
return 0; return 0;
} }
@ -3325,7 +3325,7 @@ static int block_til_ready(struct tty_struct *tty, struct file *filp,
port->blocked_open--; port->blocked_open--;
if (!retval) if (!retval)
port->flags |= ASYNC_NORMAL_ACTIVE; tty_port_set_active(port, 1);
DBGINFO(("%s block_til_ready ready, rc=%d\n", tty->driver->name, retval)); DBGINFO(("%s block_til_ready ready, rc=%d\n", tty->driver->name, retval));
return retval; return retval;

6
drivers/tty/synclinkmp.c
View File

@ -849,9 +849,9 @@ static void hangup(struct tty_struct *tty)
spin_lock_irqsave(&info->port.lock, flags); spin_lock_irqsave(&info->port.lock, flags);
info->port.count = 0; info->port.count = 0;
info->port.flags &= ~ASYNC_NORMAL_ACTIVE;
info->port.tty = NULL; info->port.tty = NULL;
spin_unlock_irqrestore(&info->port.lock, flags); spin_unlock_irqrestore(&info->port.lock, flags);
tty_port_set_active(&info->port, 1);
mutex_unlock(&info->port.mutex); mutex_unlock(&info->port.mutex);
wake_up_interruptible(&info->port.open_wait); wake_up_interruptible(&info->port.open_wait);
@ -3285,7 +3285,7 @@ static int block_til_ready(struct tty_struct *tty, struct file *filp,
if (filp->f_flags & O_NONBLOCK || tty_io_error(tty)) { if (filp->f_flags & O_NONBLOCK || tty_io_error(tty)) {
/* nonblock mode is set or port is not enabled */ /* nonblock mode is set or port is not enabled */
/* just verify that callout device is not active */ /* just verify that callout device is not active */
port->flags |= ASYNC_NORMAL_ACTIVE; tty_port_set_active(port, 1);
return 0; return 0;
} }
@ -3352,7 +3352,7 @@ static int block_til_ready(struct tty_struct *tty, struct file *filp,
__FILE__,__LINE__, tty->driver->name, port->count ); __FILE__,__LINE__, tty->driver->name, port->count );
if (!retval) if (!retval)
port->flags |= ASYNC_NORMAL_ACTIVE; tty_port_set_active(port, 1);
return retval; return retval;
} }

12
drivers/tty/tty_port.c
View File

@ -236,12 +236,12 @@ void tty_port_hangup(struct tty_port *port)
spin_lock_irqsave(&port->lock, flags); spin_lock_irqsave(&port->lock, flags);
port->count = 0; port->count = 0;
port->flags &= ~ASYNC_NORMAL_ACTIVE;
tty = port->tty; tty = port->tty;
if (tty) if (tty)
set_bit(TTY_IO_ERROR, &tty->flags); set_bit(TTY_IO_ERROR, &tty->flags);
port->tty = NULL; port->tty = NULL;
spin_unlock_irqrestore(&port->lock, flags); spin_unlock_irqrestore(&port->lock, flags);
tty_port_set_active(port, 0);
tty_port_shutdown(port, tty); tty_port_shutdown(port, tty);
tty_kref_put(tty); tty_kref_put(tty);
wake_up_interruptible(&port->open_wait); wake_up_interruptible(&port->open_wait);
@ -365,14 +365,14 @@ int tty_port_block_til_ready(struct tty_port *port,
/* if non-blocking mode is set we can pass directly to open unless /* if non-blocking mode is set we can pass directly to open unless
the port has just hung up or is in another error state */ the port has just hung up or is in another error state */
if (tty_io_error(tty)) { if (tty_io_error(tty)) {
port->flags |= ASYNC_NORMAL_ACTIVE; tty_port_set_active(port, 1);
return 0; return 0;
} }
if (filp->f_flags & O_NONBLOCK) { if (filp->f_flags & O_NONBLOCK) {
/* Indicate we are open */ /* Indicate we are open */
if (C_BAUD(tty)) if (C_BAUD(tty))
tty_port_raise_dtr_rts(port); tty_port_raise_dtr_rts(port);
port->flags |= ASYNC_NORMAL_ACTIVE; tty_port_set_active(port, 1);
return 0; return 0;
} }
@ -430,9 +430,9 @@ int tty_port_block_til_ready(struct tty_port *port,
if (!tty_hung_up_p(filp)) if (!tty_hung_up_p(filp))
port->count++; port->count++;
port->blocked_open--; port->blocked_open--;
if (retval == 0)
port->flags |= ASYNC_NORMAL_ACTIVE;
spin_unlock_irqrestore(&port->lock, flags); spin_unlock_irqrestore(&port->lock, flags);
if (retval == 0)
tty_port_set_active(port, 1);
return retval; return retval;
} }
EXPORT_SYMBOL(tty_port_block_til_ready); EXPORT_SYMBOL(tty_port_block_til_ready);
@ -514,8 +514,8 @@ void tty_port_close_end(struct tty_port *port, struct tty_struct *tty)
spin_lock_irqsave(&port->lock, flags); spin_lock_irqsave(&port->lock, flags);
wake_up_interruptible(&port->open_wait); wake_up_interruptible(&port->open_wait);
} }
port->flags &= ~ASYNC_NORMAL_ACTIVE;
spin_unlock_irqrestore(&port->lock, flags); spin_unlock_irqrestore(&port->lock, flags);
tty_port_set_active(port, 0);
} }
EXPORT_SYMBOL(tty_port_close_end); EXPORT_SYMBOL(tty_port_close_end);

12
include/linux/tty.h
View File

@ -571,6 +571,18 @@ static inline void tty_port_set_cts_flow(struct tty_port *port, bool val)
clear_bit(TTY_PORT_CTS_FLOW, &port->iflags); clear_bit(TTY_PORT_CTS_FLOW, &port->iflags);
} }
static inline bool tty_port_active(struct tty_port *port)
{
return test_bit(TTY_PORT_ACTIVE, &port->iflags);
}
static inline void tty_port_set_active(struct tty_port *port, bool val)
{
if (val)
set_bit(TTY_PORT_ACTIVE, &port->iflags);
else
clear_bit(TTY_PORT_ACTIVE, &port->iflags);
}
extern struct tty_struct *tty_port_tty_get(struct tty_port *port); extern struct tty_struct *tty_port_tty_get(struct tty_port *port);
extern void tty_port_tty_set(struct tty_port *port, struct tty_struct *tty); extern void tty_port_tty_set(struct tty_port *port, struct tty_struct *tty);

10
net/irda/ircomm/ircomm_tty.c
View File

@ -281,7 +281,7 @@ static int ircomm_tty_block_til_ready(struct ircomm_tty_cb *self,
* then make the check up front and then exit. * then make the check up front and then exit.
*/ */
if (tty_io_error(tty)) { if (tty_io_error(tty)) {
port->flags |= ASYNC_NORMAL_ACTIVE; tty_port_set_active(port, 1);
return 0; return 0;
} }
@ -289,7 +289,7 @@ static int ircomm_tty_block_til_ready(struct ircomm_tty_cb *self,
/* nonblock mode is set */ /* nonblock mode is set */
if (C_BAUD(tty)) if (C_BAUD(tty))
tty_port_raise_dtr_rts(port); tty_port_raise_dtr_rts(port);
port->flags |= ASYNC_NORMAL_ACTIVE; tty_port_set_active(port, 1);
pr_debug("%s(), O_NONBLOCK requested!\n", __func__); pr_debug("%s(), O_NONBLOCK requested!\n", __func__);
return 0; return 0;
} }
@ -365,7 +365,7 @@ static int ircomm_tty_block_til_ready(struct ircomm_tty_cb *self,
__FILE__, __LINE__, tty->driver->name, port->count); __FILE__, __LINE__, tty->driver->name, port->count);
if (!retval) if (!retval)
port->flags |= ASYNC_NORMAL_ACTIVE; tty_port_set_active(port, 1);
return retval; return retval;
} }
@ -925,7 +925,6 @@ static void ircomm_tty_hangup(struct tty_struct *tty)
ircomm_tty_shutdown(self); ircomm_tty_shutdown(self);
spin_lock_irqsave(&port->lock, flags); spin_lock_irqsave(&port->lock, flags);
port->flags &= ~ASYNC_NORMAL_ACTIVE;
if (port->tty) { if (port->tty) {
set_bit(TTY_IO_ERROR, &port->tty->flags); set_bit(TTY_IO_ERROR, &port->tty->flags);
tty_kref_put(port->tty); tty_kref_put(port->tty);
@ -933,6 +932,7 @@ static void ircomm_tty_hangup(struct tty_struct *tty)
port->tty = NULL; port->tty = NULL;
port->count = 0; port->count = 0;
spin_unlock_irqrestore(&port->lock, flags); spin_unlock_irqrestore(&port->lock, flags);
tty_port_set_active(port, 0);
wake_up_interruptible(&port->open_wait); wake_up_interruptible(&port->open_wait);
} }
@ -1267,7 +1267,7 @@ static void ircomm_tty_line_info(struct ircomm_tty_cb *self, struct seq_file *m)
seq_printf(m, "%cASYNC_LOW_LATENCY", sep); seq_printf(m, "%cASYNC_LOW_LATENCY", sep);
sep = '|'; sep = '|';
} }
if (self->port.flags & ASYNC_NORMAL_ACTIVE) { if (tty_port_active(&self->port)) {
seq_printf(m, "%cASYNC_NORMAL_ACTIVE", sep); seq_printf(m, "%cASYNC_NORMAL_ACTIVE", sep);
sep = '|'; sep = '|';
} }